From 68974918ac6c04e980bf4bfa8dabdbb8a3ea9566 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 26 Jun 2015 15:32:24 -0700 Subject: [PATCH] Donut moves to typed metadata, removing the necessity for strconv, parsing and string converstions --- pkg/storage/donut/bucket.go | 130 +++++++++++++++-------------- pkg/storage/donut/definitions.go | 23 +++-- pkg/storage/donut/donut.go | 84 +++++++++---------- pkg/storage/donut/donut_test.go | 17 ++-- pkg/storage/donut/interfaces.go | 6 +- pkg/storage/donut/object.go | 28 +++---- pkg/storage/drivers/donut/donut.go | 47 ++--------- 7 files changed, 153 insertions(+), 182 deletions(-) diff --git a/pkg/storage/donut/bucket.go b/pkg/storage/donut/bucket.go index 950ee7bc5..cb2ddd608 100644 --- a/pkg/storage/donut/bucket.go +++ b/pkg/storage/donut/bucket.go @@ -49,20 +49,18 @@ type bucket struct { } // newBucket - instantiate a new bucket -func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bucket, map[string]string, error) { +func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bucket, BucketMetadata, error) { errParams := map[string]string{ "bucketName": bucketName, "donutName": donutName, "aclType": aclType, } if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(donutName) == "" { - return bucket{}, nil, iodine.New(InvalidArgument{}, errParams) + return bucket{}, BucketMetadata{}, iodine.New(InvalidArgument{}, errParams) } - bucketMetadata := make(map[string]string) - bucketMetadata["acl"] = aclType - t := time.Now().UTC() - bucketMetadata["created"] = t.Format(time.RFC3339Nano) + b := bucket{} + t := time.Now().UTC() b.name = bucketName b.acl = aclType b.time = t @@ -70,7 +68,13 @@ func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bu b.nodes = nodes b.objects = make(map[string]object) b.lock = new(sync.RWMutex) - return b, bucketMetadata, nil + + metadata := BucketMetadata{} + metadata.Name = bucketName + metadata.ACL = aclType + metadata.Created = t + + return b, metadata, nil } func (b bucket) getBucketName() string { @@ -93,7 +97,7 @@ func (b bucket) getObjectName(fileName, diskPath, bucketPath string) (string, er return newObjectMetadata.Object, nil } -func (b bucket) GetObjectMetadata(objectName string) (*objectMetadata, error) { +func (b bucket) GetObjectMetadata(objectName string) (ObjectMetadata, error) { return b.objects[objectName].GetObjectMetadata() } @@ -183,14 +187,14 @@ func (b bucket) ReadObject(objectName string) (reader io.ReadCloser, size int64, if !ok { return nil, 0, iodine.New(ObjectNotFound{Object: objectName}, nil) } - // verify if donutObjectMetadata is readable, before we server the request - donutObjMetadata, err := object.GetDonutObjectMetadata() + // verify if sysObjectMetadata is readable, before we server the request + sysObjMetadata, err := object.GetSystemObjectMetadata() if err != nil { return nil, 0, iodine.New(err, nil) } // read and reply back to GetObject() request in a go-routine - go b.readEncodedData(b.normalizeObjectName(objectName), writer, donutObjMetadata) - return reader, donutObjMetadata.Size, nil + go b.readEncodedData(b.normalizeObjectName(objectName), writer, sysObjMetadata) + return reader, sysObjMetadata.Size, nil } // WriteObject - write a new object into bucket @@ -207,10 +211,10 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5 sumMD5 := md5.New() sum512 := sha512.New() - objectMetadata := new(objectMetadata) - donutObjectMetadata := new(donutObjectMetadata) - objectMetadata.Version = objectMetadataVersion - donutObjectMetadata.Version = donutObjectMetadataVersion + objMetadata := new(ObjectMetadata) + sysObjMetadata := new(SystemObjectMetadata) + objMetadata.Version = objectMetadataVersion + sysObjMetadata.Version = systemObjectMetadataVersion size := metadata["contentLength"] sizeInt, err := strconv.ParseInt(size, 10, 64) if err != nil { @@ -225,8 +229,8 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5 if err != nil { return "", iodine.New(err, nil) } - donutObjectMetadata.Size = totalLength - objectMetadata.Size = totalLength + sysObjMetadata.Size = totalLength + objMetadata.Size = totalLength case false: // calculate data and parity dictated by total number of writers k, m, err := b.getDataAndParity(len(writers)) @@ -239,50 +243,50 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5 return "", iodine.New(err, nil) } /// donutMetadata section - donutObjectMetadata.BlockSize = 10 * 1024 * 1024 - donutObjectMetadata.ChunkCount = chunkCount - donutObjectMetadata.DataDisks = k - donutObjectMetadata.ParityDisks = m - donutObjectMetadata.ErasureTechnique = "Cauchy" - donutObjectMetadata.Size = int64(totalLength) - // keep size inside objectMetadata as well for Object API requests - objectMetadata.Size = int64(totalLength) + sysObjMetadata.BlockSize = 10 * 1024 * 1024 + sysObjMetadata.ChunkCount = chunkCount + sysObjMetadata.DataDisks = k + sysObjMetadata.ParityDisks = m + sysObjMetadata.ErasureTechnique = "Cauchy" + sysObjMetadata.Size = int64(totalLength) + // keep size inside ObjectMetadata as well for Object API requests + objMetadata.Size = int64(totalLength) } - objectMetadata.Bucket = b.getBucketName() - objectMetadata.Object = objectName - objectMetadata.Metadata = metadata + objMetadata.Bucket = b.getBucketName() + objMetadata.Object = objectName + objMetadata.Metadata = metadata dataMD5sum := sumMD5.Sum(nil) dataSHA512sum := sum512.Sum(nil) - objectMetadata.Created = time.Now().UTC() + objMetadata.Created = time.Now().UTC() // keeping md5sum for the object in two different places // one for object storage and another is for internal use hexMD5Sum := hex.EncodeToString(dataMD5sum) hex512Sum := hex.EncodeToString(dataSHA512sum) - objectMetadata.MD5Sum = hexMD5Sum - objectMetadata.SHA512Sum = hex512Sum - donutObjectMetadata.MD5Sum = hexMD5Sum - donutObjectMetadata.SHA512Sum = hex512Sum + objMetadata.MD5Sum = hexMD5Sum + objMetadata.SHA512Sum = hex512Sum + sysObjMetadata.MD5Sum = hexMD5Sum + sysObjMetadata.SHA512Sum = hex512Sum // Verify if the written object is equal to what is expected, only if it is requested as such if strings.TrimSpace(expectedMD5Sum) != "" { - if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata.MD5Sum); err != nil { + if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objMetadata.MD5Sum); err != nil { return "", iodine.New(err, nil) } } // write donut specific metadata - if err := b.writeDonutObjectMetadata(b.normalizeObjectName(objectName), donutObjectMetadata); err != nil { + if err := b.writeSystemObjectMetadata(b.normalizeObjectName(objectName), sysObjMetadata); err != nil { return "", iodine.New(err, nil) } // write object specific metadata - if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objectMetadata); err != nil { + if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objMetadata); err != nil { return "", iodine.New(err, nil) } // close all writers, when control flow reaches here for _, writer := range writers { writer.Close() } - return objectMetadata.MD5Sum, nil + return objMetadata.MD5Sum, nil } // isMD5SumEqual - returns error if md5sum mismatches, other its `nil` @@ -305,41 +309,41 @@ func (b bucket) isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error { } // writeObjectMetadata - write additional object metadata -func (b bucket) writeObjectMetadata(objectName string, objectMetadata *objectMetadata) error { - if objectMetadata == nil { +func (b bucket) writeObjectMetadata(objectName string, objMetadata *ObjectMetadata) error { + if objMetadata == nil { return iodine.New(InvalidArgument{}, nil) } - objectMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig) + objMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig) if err != nil { return iodine.New(err, nil) } - for _, objectMetadataWriter := range objectMetadataWriters { - defer objectMetadataWriter.Close() + for _, objMetadataWriter := range objMetadataWriters { + defer objMetadataWriter.Close() } - for _, objectMetadataWriter := range objectMetadataWriters { - jenc := json.NewEncoder(objectMetadataWriter) - if err := jenc.Encode(objectMetadata); err != nil { + for _, objMetadataWriter := range objMetadataWriters { + jenc := json.NewEncoder(objMetadataWriter) + if err := jenc.Encode(objMetadata); err != nil { return iodine.New(err, nil) } } return nil } -// writeDonutObjectMetadata - write donut related object metadata -func (b bucket) writeDonutObjectMetadata(objectName string, donutObjectMetadata *donutObjectMetadata) error { - if donutObjectMetadata == nil { +// writeSystemObjectMetadata - write donut related object metadata +func (b bucket) writeSystemObjectMetadata(objectName string, sysObjMetadata *SystemObjectMetadata) error { + if sysObjMetadata == nil { return iodine.New(InvalidArgument{}, nil) } - donutObjectMetadataWriters, err := b.getDiskWriters(objectName, donutObjectMetadataConfig) + sysObjMetadataWriters, err := b.getDiskWriters(objectName, sysObjectMetadataConfig) if err != nil { return iodine.New(err, nil) } - for _, donutObjectMetadataWriter := range donutObjectMetadataWriters { - defer donutObjectMetadataWriter.Close() + for _, sysObjMetadataWriter := range sysObjMetadataWriters { + defer sysObjMetadataWriter.Close() } - for _, donutObjectMetadataWriter := range donutObjectMetadataWriters { - jenc := json.NewEncoder(donutObjectMetadataWriter) - if err := jenc.Encode(donutObjectMetadata); err != nil { + for _, sysObjMetadataWriter := range sysObjMetadataWriters { + jenc := json.NewEncoder(sysObjMetadataWriter) + if err := jenc.Encode(sysObjMetadata); err != nil { return iodine.New(err, nil) } } @@ -403,8 +407,8 @@ func (b bucket) writeEncodedData(k, m uint8, writers []io.WriteCloser, objectDat } // readEncodedData - -func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutObjMetadata *donutObjectMetadata) { - expectedMd5sum, err := hex.DecodeString(donutObjMetadata.MD5Sum) +func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, sysObjMetadata SystemObjectMetadata) { + expectedMd5sum, err := hex.DecodeString(sysObjMetadata.MD5Sum) if err != nil { writer.CloseWithError(iodine.New(err, nil)) return @@ -421,18 +425,18 @@ func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutO mwriter := io.MultiWriter(writer, hasher) switch len(readers) == 1 { case false: - if donutObjMetadata.ErasureTechnique == "" { + if sysObjMetadata.ErasureTechnique == "" { writer.CloseWithError(iodine.New(MissingErasureTechnique{}, nil)) return } - encoder, err := newEncoder(donutObjMetadata.DataDisks, donutObjMetadata.ParityDisks, donutObjMetadata.ErasureTechnique) + encoder, err := newEncoder(sysObjMetadata.DataDisks, sysObjMetadata.ParityDisks, sysObjMetadata.ErasureTechnique) if err != nil { writer.CloseWithError(iodine.New(err, nil)) return } - totalLeft := donutObjMetadata.Size - for i := 0; i < donutObjMetadata.ChunkCount; i++ { - decodedData, err := b.decodeEncodedData(totalLeft, int64(donutObjMetadata.BlockSize), readers, encoder, writer) + totalLeft := sysObjMetadata.Size + for i := 0; i < sysObjMetadata.ChunkCount; i++ { + decodedData, err := b.decodeEncodedData(totalLeft, int64(sysObjMetadata.BlockSize), readers, encoder, writer) if err != nil { writer.CloseWithError(iodine.New(err, nil)) return @@ -442,7 +446,7 @@ func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutO writer.CloseWithError(iodine.New(err, nil)) return } - totalLeft = totalLeft - int64(donutObjMetadata.BlockSize) + totalLeft = totalLeft - int64(sysObjMetadata.BlockSize) } case true: _, err := io.Copy(writer, readers[0]) diff --git a/pkg/storage/donut/definitions.go b/pkg/storage/donut/definitions.go index a8bba3fa0..bc8e7e77f 100644 --- a/pkg/storage/donut/definitions.go +++ b/pkg/storage/donut/definitions.go @@ -18,8 +18,8 @@ package donut import "time" -// objectMetadata object specific metadata per object -type objectMetadata struct { +// ObjectMetadata object specific metadata per object +type ObjectMetadata struct { // version Version string `json:"version"` @@ -37,8 +37,8 @@ type objectMetadata struct { Metadata map[string]string `json:"metadata"` } -// donutObjectMetadata container for donut specific internal metadata per object -type donutObjectMetadata struct { +// SystemObjectMetadata container for donut system specific metadata per object +type SystemObjectMetadata struct { // version Version string `json:"version"` @@ -57,14 +57,21 @@ type donutObjectMetadata struct { SHA512Sum string `json:"sys.sha512sum"` } -// donutMetadata container for donut level metadata -type donutMetadata struct { +// Metadata container for donut metadata +type Metadata struct { Version string `json:"version"` } -// bucketMetadata container for bucket level metadata -type bucketMetadata struct { +// AllBuckets container for all buckets +type AllBuckets struct { + Buckets map[string]BucketMetadata +} + +// BucketMetadata container for bucket level metadata +type BucketMetadata struct { Version string `json:"version"` + Name string `json:"name"` ACL string `json:"acl"` + Created time.Time `json:"created"` Metadata map[string]string `json:"metadata"` } diff --git a/pkg/storage/donut/donut.go b/pkg/storage/donut/donut.go index b9f7ee451..4b78d5e45 100644 --- a/pkg/storage/donut/donut.go +++ b/pkg/storage/donut/donut.go @@ -25,7 +25,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/minio/minio/pkg/iodine" ) @@ -40,17 +39,18 @@ type donut struct { // config files used inside Donut const ( - // donut object metadata and config - donutObjectMetadataConfig = "donutObjectMetadata.json" - donutConfig = "donutMetadata.json" + // donut system object metadata + sysObjectMetadataConfig = "sysObjectMetadata.json" + // donut system config + donutConfig = "donutConfig.json" // bucket, object metadata bucketMetadataConfig = "bucketMetadata.json" objectMetadataConfig = "objectMetadata.json" // versions - objectMetadataVersion = "1.0" - donutObjectMetadataVersion = "1.0" + objectMetadataVersion = "1.0.0" + systemObjectMetadataVersion = "1.0.0" ) // attachDonutNode - wrapper function to instantiate a new node for associatedt donut @@ -98,24 +98,24 @@ func (dt donut) MakeBucket(bucket, acl string) error { } // GetBucketMetadata - get bucket metadata -func (dt donut) GetBucketMetadata(bucket string) (map[string]string, error) { +func (dt donut) GetBucketMetadata(bucketName string) (BucketMetadata, error) { dt.lock.RLock() defer dt.lock.RUnlock() if err := dt.listDonutBuckets(); err != nil { - return nil, iodine.New(err, nil) + return BucketMetadata{}, iodine.New(err, nil) } - if _, ok := dt.buckets[bucket]; !ok { - return nil, iodine.New(BucketNotFound{Bucket: bucket}, nil) + if _, ok := dt.buckets[bucketName]; !ok { + return BucketMetadata{}, iodine.New(BucketNotFound{Bucket: bucketName}, nil) } metadata, err := dt.getDonutBucketMetadata() if err != nil { - return nil, iodine.New(err, nil) + return BucketMetadata{}, iodine.New(err, nil) } - return metadata[bucket], nil + return metadata.Buckets[bucketName], nil } // SetBucketMetadata - set bucket metadata -func (dt donut) SetBucketMetadata(bucket string, bucketMetadata map[string]string) error { +func (dt donut) SetBucketMetadata(bucketName string, bucketMetadata map[string]string) error { dt.lock.Lock() defer dt.lock.Unlock() if err := dt.listDonutBuckets(); err != nil { @@ -125,28 +125,31 @@ func (dt donut) SetBucketMetadata(bucket string, bucketMetadata map[string]strin if err != nil { return iodine.New(err, nil) } - oldBucketMetadata := metadata[bucket] - // TODO ignore rest of the keys for now, only mutable data is "acl" - oldBucketMetadata["acl"] = bucketMetadata["acl"] - metadata[bucket] = oldBucketMetadata + oldBucketMetadata := metadata.Buckets[bucketName] + acl, ok := bucketMetadata["acl"] + if !ok { + return iodine.New(InvalidArgument{}, nil) + } + oldBucketMetadata.ACL = acl + metadata.Buckets[bucketName] = oldBucketMetadata return dt.setDonutBucketMetadata(metadata) } // ListBuckets - return list of buckets -func (dt donut) ListBuckets() (metadata map[string]map[string]string, err error) { +func (dt donut) ListBuckets() (map[string]BucketMetadata, error) { dt.lock.RLock() defer dt.lock.RUnlock() if err := dt.listDonutBuckets(); err != nil { return nil, iodine.New(err, nil) } - metadata, err = dt.getDonutBucketMetadata() + metadata, err := dt.getDonutBucketMetadata() if err != nil { // intentionally left out the error when Donut is empty // but we need to revisit this area in future - since we need // to figure out between acceptable and unacceptable errors - return make(map[string]map[string]string), nil + return make(map[string]BucketMetadata), nil } - return metadata, nil + return metadata.Buckets, nil } // ListObjects - return list of objects @@ -233,7 +236,7 @@ func (dt donut) GetObject(bucket, object string) (reader io.ReadCloser, size int } // GetObjectMetadata - get object metadata -func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, error) { +func (dt donut) GetObjectMetadata(bucket, object string) (ObjectMetadata, error) { dt.lock.RLock() defer dt.lock.RUnlock() errParams := map[string]string{ @@ -241,10 +244,10 @@ func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, err "object": object, } if err := dt.listDonutBuckets(); err != nil { - return nil, iodine.New(err, errParams) + return ObjectMetadata{}, iodine.New(err, errParams) } if _, ok := dt.buckets[bucket]; !ok { - return nil, iodine.New(BucketNotFound{Bucket: bucket}, errParams) + return ObjectMetadata{}, iodine.New(BucketNotFound{Bucket: bucket}, errParams) } // // there is a potential issue here, if the object comes after the truncated list @@ -253,26 +256,14 @@ func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, err // will fix it when we bring in persistent json into Donut - TODO objectList, _, _, err := dt.buckets[bucket].ListObjects("", "", "", 1000) if err != nil { - return nil, iodine.New(err, errParams) + return ObjectMetadata{}, iodine.New(err, errParams) } for _, objectName := range objectList { if objectName == object { - objectMetadataMap := make(map[string]string) - objectMetadata, err := dt.buckets[bucket].GetObjectMetadata(object) - if err != nil { - return nil, iodine.New(err, nil) - } - objectMetadataMap["created"] = objectMetadata.Created.Format(time.RFC3339Nano) - objectMetadataMap["size"] = strconv.FormatInt(objectMetadata.Size, 10) - objectMetadataMap["md5"] = objectMetadata.MD5Sum - objectMetadataMap["version"] = objectMetadata.Version - for k, v := range objectMetadata.Metadata { - objectMetadataMap[k] = v - } - return objectMetadataMap, nil + return dt.buckets[bucket].GetObjectMetadata(object) } } - return nil, iodine.New(ObjectNotFound{Object: object}, errParams) + return ObjectMetadata{}, iodine.New(ObjectNotFound{Object: object}, errParams) } // getDiskWriters - @@ -315,7 +306,7 @@ func (dt donut) getBucketMetadataReaders() ([]io.ReadCloser, error) { } // -func (dt donut) setDonutBucketMetadata(metadata map[string]map[string]string) error { +func (dt donut) setDonutBucketMetadata(metadata *AllBuckets) error { writers, err := dt.getBucketMetadataWriters() if err != nil { return iodine.New(err, nil) @@ -332,8 +323,8 @@ func (dt donut) setDonutBucketMetadata(metadata map[string]map[string]string) er return nil } -func (dt donut) getDonutBucketMetadata() (map[string]map[string]string, error) { - metadata := make(map[string]map[string]string) +func (dt donut) getDonutBucketMetadata() (*AllBuckets, error) { + metadata := new(AllBuckets) readers, err := dt.getBucketMetadataReaders() if err != nil { return nil, iodine.New(err, nil) @@ -343,7 +334,7 @@ func (dt donut) getDonutBucketMetadata() (map[string]map[string]string, error) { } for _, reader := range readers { jenc := json.NewDecoder(reader) - if err := jenc.Decode(&metadata); err != nil { + if err := jenc.Decode(metadata); err != nil { return nil, iodine.New(err, nil) } } @@ -380,8 +371,9 @@ func (dt donut) makeDonutBucket(bucketName, acl string) error { metadata, err := dt.getDonutBucketMetadata() if err != nil { if os.IsNotExist(iodine.ToError(err)) { - metadata := make(map[string]map[string]string) - metadata[bucketName] = bucketMetadata + metadata := new(AllBuckets) + metadata.Buckets = make(map[string]BucketMetadata) + metadata.Buckets[bucketName] = bucketMetadata err = dt.setDonutBucketMetadata(metadata) if err != nil { return iodine.New(err, nil) @@ -390,7 +382,7 @@ func (dt donut) makeDonutBucket(bucketName, acl string) error { } return iodine.New(err, nil) } - metadata[bucketName] = bucketMetadata + metadata.Buckets[bucketName] = bucketMetadata err = dt.setDonutBucketMetadata(metadata) if err != nil { return iodine.New(err, nil) diff --git a/pkg/storage/donut/donut_test.go b/pkg/storage/donut/donut_test.go index 83243b932..476ef6b76 100644 --- a/pkg/storage/donut/donut_test.go +++ b/pkg/storage/donut/donut_test.go @@ -26,7 +26,6 @@ import ( "path/filepath" "strconv" "testing" - "time" . "github.com/minio/check" ) @@ -113,7 +112,7 @@ func (s *MySuite) TestMakeBucketAndList(c *C) { buckets, err := donut.ListBuckets() c.Assert(err, IsNil) c.Assert(len(buckets), Equals, 1) - c.Assert(buckets["foo"]["acl"], Equals, "private") + c.Assert(buckets["foo"].ACL, Equals, "private") } // test re-create bucket @@ -202,9 +201,9 @@ func (s *MySuite) TestNewObjectMetadata(c *C) { objectMetadata, err := donut.GetObjectMetadata("foo", "obj") c.Assert(err, IsNil) - c.Assert(objectMetadata["contentType"], Equals, metadata["contentType"]) - c.Assert(objectMetadata["foo"], Equals, metadata["foo"]) - c.Assert(objectMetadata["hello"], Equals, metadata["hello"]) + c.Assert(objectMetadata.Metadata["contentType"], Equals, metadata["contentType"]) + c.Assert(objectMetadata.Metadata["foo"], Equals, metadata["foo"]) + c.Assert(objectMetadata.Metadata["hello"], Equals, metadata["hello"]) } // test create object fails without name @@ -255,11 +254,9 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) { actualMetadata, err := donut.GetObjectMetadata("foo", "obj") c.Assert(err, IsNil) - c.Assert(expectedMd5Sum, Equals, actualMetadata["md5"]) - c.Assert(strconv.Itoa(len(data)), Equals, actualMetadata["size"]) - c.Assert("1.0", Equals, actualMetadata["version"]) - _, err = time.Parse(time.RFC3339Nano, actualMetadata["created"]) - c.Assert(err, IsNil) + c.Assert(expectedMd5Sum, Equals, actualMetadata.MD5Sum) + c.Assert(int64(len(data)), Equals, actualMetadata.Size) + c.Assert("1.0.0", Equals, actualMetadata.Version) } // test list objects diff --git a/pkg/storage/donut/interfaces.go b/pkg/storage/donut/interfaces.go index 668287b9e..5cd8648c8 100644 --- a/pkg/storage/donut/interfaces.go +++ b/pkg/storage/donut/interfaces.go @@ -29,9 +29,9 @@ type Donut interface { // ObjectStorage is a donut object storage interface type ObjectStorage interface { // Storage service operations - GetBucketMetadata(bucket string) (map[string]string, error) + GetBucketMetadata(bucket string) (BucketMetadata, error) SetBucketMetadata(bucket string, metadata map[string]string) error - ListBuckets() (map[string]map[string]string, error) + ListBuckets() (map[string]BucketMetadata, error) MakeBucket(bucket, acl string) error // Bucket operations @@ -39,7 +39,7 @@ type ObjectStorage interface { // Object operations GetObject(bucket, object string) (io.ReadCloser, int64, error) - GetObjectMetadata(bucket, object string) (map[string]string, error) + GetObjectMetadata(bucket, object string) (ObjectMetadata, error) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error) } diff --git a/pkg/storage/donut/object.go b/pkg/storage/donut/object.go index 098cfc007..85c16616a 100644 --- a/pkg/storage/donut/object.go +++ b/pkg/storage/donut/object.go @@ -41,26 +41,26 @@ func newObject(objectName, p string) (object, error) { return o, nil } -func (o object) GetObjectMetadata() (*objectMetadata, error) { - objectMetadata := new(objectMetadata) - objectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, objectMetadataConfig)) +func (o object) GetObjectMetadata() (ObjectMetadata, error) { + objMetadata := ObjectMetadata{} + objMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, objectMetadataConfig)) if err != nil { - return nil, iodine.New(ObjectNotFound{Object: o.name}, nil) + return ObjectMetadata{}, iodine.New(ObjectNotFound{Object: o.name}, nil) } - if err := json.Unmarshal(objectMetadataBytes, objectMetadata); err != nil { - return nil, iodine.New(err, nil) + if err := json.Unmarshal(objMetadataBytes, &objMetadata); err != nil { + return ObjectMetadata{}, iodine.New(err, nil) } - return objectMetadata, nil + return objMetadata, nil } -func (o object) GetDonutObjectMetadata() (*donutObjectMetadata, error) { - donutObjectMetadata := new(donutObjectMetadata) - donutObjectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, donutObjectMetadataConfig)) +func (o object) GetSystemObjectMetadata() (SystemObjectMetadata, error) { + sysObjMetadata := SystemObjectMetadata{} + sysObjMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, sysObjectMetadataConfig)) if err != nil { - return nil, iodine.New(ObjectNotFound{Object: o.name}, nil) + return SystemObjectMetadata{}, iodine.New(ObjectNotFound{Object: o.name}, nil) } - if err := json.Unmarshal(donutObjectMetadataBytes, donutObjectMetadata); err != nil { - return nil, iodine.New(err, nil) + if err := json.Unmarshal(sysObjMetadataBytes, &sysObjMetadata); err != nil { + return SystemObjectMetadata{}, iodine.New(err, nil) } - return donutObjectMetadata, nil + return sysObjMetadata, nil } diff --git a/pkg/storage/drivers/donut/donut.go b/pkg/storage/drivers/donut/donut.go index 48572df7e..69e6ae1b1 100644 --- a/pkg/storage/drivers/donut/donut.go +++ b/pkg/storage/drivers/donut/donut.go @@ -26,7 +26,6 @@ import ( "strconv" "strings" "sync" - "time" "io/ioutil" @@ -141,13 +140,9 @@ func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error) return nil, err } for name, metadata := range buckets { - created, err := time.Parse(time.RFC3339Nano, metadata["created"]) - if err != nil { - return nil, iodine.New(err, nil) - } result := drivers.BucketMetadata{ Name: name, - Created: created, + Created: metadata.Created, } results = append(results, result) } @@ -195,18 +190,10 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat if err != nil { return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucketName}, nil) } - created, err := time.Parse(time.RFC3339Nano, metadata["created"]) - if err != nil { - return drivers.BucketMetadata{}, iodine.New(err, nil) - } - acl, ok := metadata["acl"] - if !ok { - return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{}, nil) - } bucketMetadata := drivers.BucketMetadata{ Name: bucketName, - Created: created, - ACL: drivers.BucketACL(acl), + Created: metadata.Created, + ACL: drivers.BucketACL(metadata.ACL), } return bucketMetadata, nil } @@ -332,22 +319,14 @@ func (d donutDriver) GetObjectMetadata(bucketName, objectName string) (drivers.O Object: objectName, }, errParams) } - created, err := time.Parse(time.RFC3339Nano, metadata["created"]) - if err != nil { - return drivers.ObjectMetadata{}, iodine.New(err, errParams) - } - size, err := strconv.ParseInt(metadata["size"], 10, 64) - if err != nil { - return drivers.ObjectMetadata{}, iodine.New(err, errParams) - } objectMetadata := drivers.ObjectMetadata{ Bucket: bucketName, Key: objectName, - ContentType: metadata["contentType"], - Created: created, - Md5: metadata["md5"], - Size: size, + ContentType: metadata.Metadata["contentType"], + Created: metadata.Created, + Md5: metadata.MD5Sum, + Size: metadata.Size, } return objectMetadata, nil } @@ -390,18 +369,10 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso if err != nil { return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams) } - t, err := time.Parse(time.RFC3339Nano, objectMetadata["created"]) - if err != nil { - return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil) - } - size, err := strconv.ParseInt(objectMetadata["size"], 10, 64) - if err != nil { - return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil) - } metadata := drivers.ObjectMetadata{ Key: objectName, - Created: t, - Size: size, + Created: objectMetadata.Created, + Size: objectMetadata.Size, } results = append(results, metadata) }