MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-26 07:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Adding content type to backend storage drivers

Browse Source
This commit is contained in:
Frederick F. Kautz IV 2015-02-04 17:32:40 -08:00
parent e5cd3dcb7c
commit 288bc3fcc7
6 changed files with 144 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/api/minioapi/minioapi.go
View File

@ -192,7 +192,7 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques
vars := mux.Vars(req)
bucket := vars["bucket"]
object := vars["object"]
err := server.storage.StoreObject(bucket, object, req.Body)
err := server.storage.StoreObject(bucket, object, "", req.Body)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))

12
pkg/api/minioapi/minioapi_test.go
View File

@ -57,7 +57,7 @@ func (s *MySuite) TestEmptyObject(c *C) {
buffer := bytes.NewBufferString("")
storage.StoreBucket("bucket")
storage.StoreObject("bucket", "object", buffer)
storage.StoreObject("bucket", "object", "", buffer)
response, err := http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)
@ -82,7 +82,7 @@ func (s *MySuite) TestObject(c *C) {
buffer := bytes.NewBufferString("hello world")
storage.StoreBucket("bucket")
storage.StoreObject("bucket", "object", buffer)
storage.StoreObject("bucket", "object", "", buffer)
response, err := http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)
@ -108,9 +108,9 @@ func (s *MySuite) TestMultipleObjects(c *C) {
buffer3 := bytes.NewBufferString("hello three")
storage.StoreBucket("bucket")
storage.StoreObject("bucket", "object1", buffer1)
storage.StoreObject("bucket", "object2", buffer2)
storage.StoreObject("bucket", "object3", buffer3)
storage.StoreObject("bucket", "object1", "", buffer1)
storage.StoreObject("bucket", "object2", "", buffer2)
storage.StoreObject("bucket", "object3", "", buffer3)
// test non-existant object
response, err := http.Get(testServer.URL + "/bucket/object")
@ -200,7 +200,7 @@ func (s *MySuite) TestHeader(c *C) {
buffer := bytes.NewBufferString("hello world")
storage.StoreBucket("bucket")
storage.StoreObject("bucket", "object", buffer)
storage.StoreObject("bucket", "object", "", buffer)
response, err = http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)

96
pkg/storage/fs/fs.go
View File

@ -1,6 +1,8 @@
package fs
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"os"
@ -16,6 +18,10 @@ type storage struct {
writeLock sync.Mutex
}
type SerializedMetadata struct {
ContentType string
}
type MkdirFailedError struct{}
func (self MkdirFailedError) Error() string {
@ -146,12 +152,41 @@ func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorag
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
}
_, err = os.Stat(objectPath + "$metadata")
if os.IsNotExist(err) {
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
}
file, err := os.Open(objectPath + "$metadata")
defer file.Close()
if err != nil {
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
}
metadataBuffer, err := ioutil.ReadAll(file)
if err != nil {
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
}
var deserializedMetadata SerializedMetadata
err = json.Unmarshal(metadataBuffer, &deserializedMetadata)
if err != nil {
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
}
contentType := "application/octet-stream"
if deserializedMetadata.ContentType != "" {
contentType = deserializedMetadata.ContentType
}
contentType = strings.TrimSpace(contentType)
metadata := mstorage.ObjectMetadata{
Bucket: bucket,
Key: object,
Created: stat.ModTime(),
Size: stat.Size(),
ETag: bucket + "#" + object,
Bucket: bucket,
Key: object,
Created: stat.ModTime(),
Size: stat.Size(),
ETag: bucket + "#" + object,
ContentType: contentType,
}
return metadata, nil
@ -179,24 +214,26 @@ func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorag
var metadataList []mstorage.ObjectMetadata
for _, file := range files {
if len(metadataList) >= count {
return metadataList, true, nil
}
if strings.HasPrefix(file.Name(), prefix) {
metadata := mstorage.ObjectMetadata{
Bucket: bucket,
Key: file.Name(),
Created: file.ModTime(),
Size: file.Size(),
ETag: bucket + "#" + file.Name(),
if !strings.HasSuffix(file.Name(), "$metadata") {
if len(metadataList) >= count {
return metadataList, true, nil
}
if strings.HasPrefix(file.Name(), prefix) {
metadata := mstorage.ObjectMetadata{
Bucket: bucket,
Key: file.Name(),
Created: file.ModTime(),
Size: file.Size(),
ETag: bucket + "#" + file.Name(),
}
metadataList = append(metadataList, metadata)
}
metadataList = append(metadataList, metadata)
}
}
return metadataList, false, nil
}
func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
// TODO Commits should stage then move instead of writing directly
storage.writeLock.Lock()
defer storage.writeLock.Unlock()
@ -216,6 +253,12 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key}
}
// verify content type
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
// get object path
objectPath := path.Join(storage.root, bucket, key)
objectDir := path.Dir(objectPath)
@ -246,5 +289,24 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
return mstorage.EmbedError(bucket, key, err)
}
// serialize metadata to json
metadataBuffer, err := json.Marshal(SerializedMetadata{ContentType: contentType})
if err != nil {
return mstorage.EmbedError(bucket, key, err)
}
//
file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
defer file.Close()
if err != nil {
return mstorage.EmbedError(bucket, key, err)
}
_, err = io.Copy(file, bytes.NewBuffer(metadataBuffer))
if err != nil {
return mstorage.EmbedError(bucket, key, err)
}
return nil
}

21
pkg/storage/inmemory/inmemory.go
View File

@ -41,7 +41,7 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st
}
}
func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
objectKey := bucket + ":" + key
if _, ok := storage.bucketdata[bucket]; ok == false {
@ -51,17 +51,26 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
if _, ok := storage.objectdata[objectKey]; ok == true {
return mstorage.ObjectExists{Bucket: bucket, Key: key}
}
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
var bytesBuffer bytes.Buffer
var newObject = storedObject{}
if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
size := bytesBuffer.Len()
etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes()))
newObject.metadata = mstorage.ObjectMetadata{
Bucket: bucket,
Key: key,
Created: time.Now(),
Size: int64(size),
ETag: etag,
Bucket: bucket,
Key: key,
ContentType: contentType,
Created: time.Now(),
ETag: etag,
Size: int64(size),
}
newObject.data = bytesBuffer.Bytes()
}

14
pkg/storage/storage.go
View File

@ -31,7 +31,7 @@ type Storage interface {
CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error)
GetObjectMetadata(bucket string, object string) (ObjectMetadata, error)
ListObjects(bucket, prefix string, count int) ([]ObjectMetadata, bool, error)
StoreObject(bucket string, key string, data io.Reader) error
StoreObject(bucket string, key string, contentType string, data io.Reader) error
}
type BucketMetadata struct {
@ -40,11 +40,13 @@ type BucketMetadata struct {
}
type ObjectMetadata struct {
Bucket string
Key string
Created time.Time
Size int64
ETag string
Bucket string
Key string
ContentType string
Created time.Time
ETag string
Size int64
}
func IsValidBucket(bucket string) bool {

45
pkg/storage/storage_api_suite.go
View File

@ -21,6 +21,7 @@ func APITestSuite(c *C, create func() Storage) {
testListObjectsTestsForNonExistantBucket(c, create)
testNonExistantObjectInBucket(c, create)
testGetDirectoryReturnsObjectNotFound(c, create)
testDefaultContentType(c, create)
}
func testCreateBucket(c *C, create func() Storage) {
@ -40,7 +41,7 @@ func testMultipleObjectCreation(c *C, create func() Storage) {
}
key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString)
err := storage.StoreObject("bucket", key, bytes.NewBufferString(randomString))
err := storage.StoreObject("bucket", key, "", bytes.NewBufferString(randomString))
c.Assert(err, IsNil)
}
@ -72,7 +73,7 @@ func testPaging(c *C, create func() Storage) {
// check before paging occurs
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
storage.StoreObject("bucket", key, bytes.NewBufferString(key))
storage.StoreObject("bucket", key, "", bytes.NewBufferString(key))
objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
c.Assert(len(objects), Equals, i+1)
c.Assert(isTruncated, Equals, false)
@ -81,7 +82,7 @@ func testPaging(c *C, create func() Storage) {
// check after paging occurs pages work
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
storage.StoreObject("bucket", key, bytes.NewBufferString(key))
storage.StoreObject("bucket", key, "", bytes.NewBufferString(key))
objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
c.Assert(len(objects), Equals, 5)
c.Assert(isTruncated, Equals, true)
@ -89,8 +90,8 @@ func testPaging(c *C, create func() Storage) {
}
// check paging with prefix at end returns less objects
{
storage.StoreObject("bucket", "newPrefix", bytes.NewBufferString("prefix1"))
storage.StoreObject("bucket", "newPrefix2", bytes.NewBufferString("prefix2"))
storage.StoreObject("bucket", "newPrefix", "", bytes.NewBufferString("prefix1"))
storage.StoreObject("bucket", "newPrefix2", "", bytes.NewBufferString("prefix2"))
objects, isTruncated, err = storage.ListObjects("bucket", "new", 5)
c.Assert(len(objects), Equals, 2)
}
@ -124,9 +125,9 @@ func testPaging(c *C, create func() Storage) {
func testObjectOverwriteFails(c *C, create func() Storage) {
storage := create()
storage.StoreBucket("bucket")
err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one"))
c.Assert(err, IsNil)
err = storage.StoreObject("bucket", "object", bytes.NewBufferString("three"))
err = storage.StoreObject("bucket", "object", "", bytes.NewBufferString("three"))
c.Assert(err, Not(IsNil))
var bytesBuffer bytes.Buffer
length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "object")
@ -137,7 +138,7 @@ func testObjectOverwriteFails(c *C, create func() Storage) {
func testNonExistantBucketOperations(c *C, create func() Storage) {
storage := create()
err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one"))
c.Assert(err, Not(IsNil))
}
@ -153,7 +154,7 @@ func testPutObjectInSubdir(c *C, create func() Storage) {
storage := create()
err := storage.StoreBucket("bucket")
c.Assert(err, IsNil)
err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world"))
err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world"))
c.Assert(err, IsNil)
var bytesBuffer bytes.Buffer
length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "dir1/dir2/object")
@ -250,7 +251,7 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) {
err := storage.StoreBucket("bucket")
c.Assert(err, IsNil)
err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world"))
err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world"))
c.Assert(err, IsNil)
var byteBuffer bytes.Buffer
@ -287,3 +288,27 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) {
}
c.Assert(len(byteBuffer2.Bytes()), Equals, 0)
}
func testDefaultContentType(c *C, create func() Storage) {
storage := create()
err := storage.StoreBucket("bucket")
c.Assert(err, IsNil)
// test empty
err = storage.StoreObject("bucket", "one", "", bytes.NewBufferString("one"))
metadata, err := storage.GetObjectMetadata("bucket", "one")
c.Assert(err, IsNil)
c.Assert(metadata.ContentType, Equals, "application/octet-stream")
// test custom
storage.StoreObject("bucket", "two", "application/text", bytes.NewBufferString("two"))
metadata, err = storage.GetObjectMetadata("bucket", "two")
c.Assert(err, IsNil)
c.Assert(metadata.ContentType, Equals, "application/text")
// test trim space
storage.StoreObject("bucket", "three", "\tapplication/json ", bytes.NewBufferString("three"))
metadata, err = storage.GetObjectMetadata("bucket", "three")
c.Assert(err, IsNil)
c.Assert(metadata.ContentType, Equals, "application/json")
}