From 53669a0854b256dc548a60724d852dd229c527c0 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 26 Feb 2015 17:23:42 -0800 Subject: [PATCH] Implement delimiter, path prefix --- pkg/api/minioapi/api_test.go | 16 +++-- pkg/api/minioapi/bucket_handlers.go | 10 +-- pkg/api/minioapi/definitions.go | 17 +++-- pkg/api/minioapi/object_handlers.go | 2 +- pkg/api/minioapi/resources.go | 26 +++----- pkg/storage/fs/fs.go | 99 ++++++++++++++++++++++------- pkg/storage/inmemory/inmemory.go | 14 ++-- pkg/storage/storage.go | 25 +++++++- pkg/storage/storage_api_suite.go | 37 +++++++---- pkg/utils/crypto/sha256/sha256.go | 2 +- 10 files changed, 168 insertions(+), 80 deletions(-) diff --git a/pkg/api/minioapi/api_test.go b/pkg/api/minioapi/api_test.go index b4247f174..de37a7281 100644 --- a/pkg/api/minioapi/api_test.go +++ b/pkg/api/minioapi/api_test.go @@ -28,6 +28,7 @@ import ( "testing" "time" + mstorage "github.com/minio-io/minio/pkg/storage" "github.com/minio-io/minio/pkg/storage/inmemory" . "gopkg.in/check.v1" ) @@ -243,9 +244,13 @@ func (s *MySuite) TestPutObject(c *C) { testServer := httptest.NewServer(httpHandler) defer testServer.Close() - objects, isTruncated, err := storage.ListObjects("bucket", "", 1000) + resources := mstorage.BucketResourcesMetadata{} + + resources.Maxkeys = 1000 + resources.Prefix = "" + objects, resources, err := storage.ListObjects("bucket", resources) c.Assert(len(objects), Equals, 0) - c.Assert(isTruncated, Equals, false) + c.Assert(resources.IsTruncated, Equals, false) c.Assert(err, Not(IsNil)) date1 := time.Now() @@ -268,9 +273,12 @@ func (s *MySuite) TestPutObject(c *C) { date2 := time.Now() - objects, isTruncated, err = storage.ListObjects("bucket", "", 1000) + resources.Maxkeys = 1000 + resources.Prefix = "" + + objects, resources, err = storage.ListObjects("bucket", resources) c.Assert(len(objects), Equals, 1) - c.Assert(isTruncated, Equals, false) + c.Assert(resources.IsTruncated, Equals, false) c.Assert(err, IsNil) var writer bytes.Buffer diff --git a/pkg/api/minioapi/bucket_handlers.go b/pkg/api/minioapi/bucket_handlers.go index 8d73536a6..cd7ace526 100644 --- a/pkg/api/minioapi/bucket_handlers.go +++ b/pkg/api/minioapi/bucket_handlers.go @@ -35,17 +35,17 @@ func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Requ bucket := vars["bucket"] resources := getBucketResources(req.URL.Query()) - if resources.policy == true { + if resources.Policy == true { server.getBucketPolicyHandler(w, req) return } acceptsContentType := getContentType(req) - objects, isTruncated, err := server.storage.ListObjects(bucket, resources.prefix, 1000) + objects, resources, err := server.storage.ListObjects(bucket, resources) switch err := err.(type) { case nil: // success { - response := generateObjectsListResult(bucket, objects, isTruncated) + response := generateObjectsListResult(bucket, objects, resources.IsTruncated) w.Write(writeObjectHeadersAndResponse(w, response, acceptsContentType)) } case mstorage.BucketNotFound: @@ -74,7 +74,7 @@ func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Requ case mstorage.ObjectNameInvalid: { error := errorCodeError(NoSuchKey) - errorResponse := getErrorResponse(error, resources.prefix) + errorResponse := getErrorResponse(error, resources.Prefix) w.WriteHeader(error.HttpStatusCode) w.Write(writeErrorResponse(w, errorResponse, acceptsContentType)) } @@ -122,7 +122,7 @@ func (server *minioApi) putBucketHandler(w http.ResponseWriter, req *http.Reques err := server.storage.StoreBucket(bucket) resources := getBucketResources(req.URL.Query()) - if resources.policy == true { + if resources.Policy == true { server.putBucketPolicyHandler(w, req) return } diff --git a/pkg/api/minioapi/definitions.go b/pkg/api/minioapi/definitions.go index cca9a8687..8192f05b1 100644 --- a/pkg/api/minioapi/definitions.go +++ b/pkg/api/minioapi/definitions.go @@ -27,12 +27,17 @@ const ( // Object list response format type ObjectListResponse struct { - XMLName xml.Name `xml:"ListBucketResult" json:"-"` - Name string - Marker string - MaxKeys int - IsTruncated bool - Contents []*Item `xml:,innerxml` + XMLName xml.Name `xml:"ListBucketResult" json:"-"` + Name string + Prefix string + Marker string + MaxKeys int + Delimiter string + IsTruncated bool + Contents []*Item `xml:,innerxml` + CommonPrefixes struct { + Prefix string + } `xml:,innerxml` } // Bucket list response format diff --git a/pkg/api/minioapi/object_handlers.go b/pkg/api/minioapi/object_handlers.go index 9fe0c1ef5..3f0a74474 100644 --- a/pkg/api/minioapi/object_handlers.go +++ b/pkg/api/minioapi/object_handlers.go @@ -130,7 +130,7 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques object = vars["object"] resources := getBucketResources(req.URL.Query()) - if resources.policy == true && object == "" { + if resources.Policy == true && object == "" { server.putBucketPolicyHandler(w, req) return } diff --git a/pkg/api/minioapi/resources.go b/pkg/api/minioapi/resources.go index 2ccd4c633..a38d2e53f 100644 --- a/pkg/api/minioapi/resources.go +++ b/pkg/api/minioapi/resources.go @@ -19,32 +19,24 @@ package minioapi import ( "net/url" "strconv" + + mstorage "github.com/minio-io/minio/pkg/storage" ) -// support bucket resources go here -type bucketResources struct { - prefix string - marker string - maxkeys int - policy bool - delimiter string - // uploads bool - TODO implemented with multipart support -} - // parse bucket url queries -func getBucketResources(values url.Values) (v bucketResources) { +func getBucketResources(values url.Values) (v mstorage.BucketResourcesMetadata) { for key, value := range values { switch true { case key == "prefix": - v.prefix = value[0] + v.Prefix = value[0] case key == "marker": - v.marker = value[0] - case key == "maxkeys": - v.maxkeys, _ = strconv.Atoi(value[0]) + v.Marker = value[0] + case key == "max-keys": + v.Maxkeys, _ = strconv.Atoi(value[0]) case key == "policy": - v.policy = true + v.Policy = true case key == "delimiter": - v.delimiter = value[0] + v.Delimiter = value[0] } } return diff --git a/pkg/storage/fs/fs.go b/pkg/storage/fs/fs.go index 3109ec855..cecf17fa1 100644 --- a/pkg/storage/fs/fs.go +++ b/pkg/storage/fs/fs.go @@ -23,6 +23,8 @@ import ( "io/ioutil" "os" "path" + "path/filepath" + "sort" "strings" "sync" @@ -291,45 +293,96 @@ func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorag return metadata, nil } -func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorage.ObjectMetadata, bool, error) { - if mstorage.IsValidBucket(bucket) == false { - return []mstorage.ObjectMetadata{}, false, mstorage.BucketNameInvalid{Bucket: bucket} +type Path struct { + files map[string]os.FileInfo + root string +} + +func (p *Path) getAllFiles(path string, fl os.FileInfo, err error) error { + if err != nil { + return err } - if mstorage.IsValidObject(prefix) == false { - return []mstorage.ObjectMetadata{}, false, mstorage.ObjectNameInvalid{Bucket: bucket, Object: prefix} + if fl.Mode().IsRegular() { + if strings.HasSuffix(path, "$metadata") { + return nil + } + _p := strings.Split(path, p.root+"/") + if len(_p) > 1 { + p.files[_p[1]] = fl + } + } + return nil +} + +type ByObjectKey []mstorage.ObjectMetadata + +func (b ByObjectKey) Len() int { return len(b) } +func (b ByObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b ByObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key } + +func (storage *storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) { + p := Path{} + p.files = make(map[string]os.FileInfo) + + if mstorage.IsValidBucket(bucket) == false { + return []mstorage.ObjectMetadata{}, resources, mstorage.BucketNameInvalid{Bucket: bucket} + } + if mstorage.IsValidObject(resources.Prefix) == false { + return []mstorage.ObjectMetadata{}, resources, mstorage.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix} } rootPrefix := path.Join(storage.root, bucket) - // check bucket exists if _, err := os.Stat(rootPrefix); os.IsNotExist(err) { - return []mstorage.ObjectMetadata{}, false, mstorage.BucketNotFound{Bucket: bucket} + return []mstorage.ObjectMetadata{}, resources, mstorage.BucketNotFound{Bucket: bucket} } - files, err := ioutil.ReadDir(rootPrefix) + p.root = rootPrefix + err := filepath.Walk(rootPrefix, p.getAllFiles) if err != nil { - return []mstorage.ObjectMetadata{}, false, mstorage.EmbedError("bucket", "", err) + return []mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err) } var metadataList []mstorage.ObjectMetadata - for _, file := range files { - if !strings.HasSuffix(file.Name(), "$metadata") { - if len(metadataList) >= count { - return metadataList, true, nil + for name, file := range p.files { + if len(metadataList) >= resources.Maxkeys { + resources.IsTruncated = true + goto ret + } + // TODO handle resources.Marker + if resources.Delimiter != "" { + metadata := mstorage.ObjectMetadata{ + Bucket: bucket, + Maxkeys: resources.Maxkeys, + Prefix: resources.Prefix, + Marker: resources.Marker, + Delimiter: resources.Delimiter, } - if strings.HasPrefix(file.Name(), prefix) { - metadata := mstorage.ObjectMetadata{ - Bucket: bucket, - Key: file.Name(), - Created: file.ModTime(), - Size: file.Size(), - ETag: bucket + "#" + file.Name(), - } - metadataList = append(metadataList, metadata) + metadataList = append(metadataList, metadata) + } + if resources.Delimiter != "" && strings.HasPrefix(name, resources.Prefix) { + metadata := mstorage.ObjectMetadata{} + metadataList = append(metadataList, metadata) + } + if strings.HasPrefix(name, resources.Prefix) { + metadata := mstorage.ObjectMetadata{ + Bucket: bucket, + Maxkeys: resources.Maxkeys, + Prefix: resources.Prefix, + Marker: resources.Marker, + Delimiter: resources.Delimiter, + Key: name, + Created: file.ModTime(), + Size: file.Size(), + ETag: bucket + "#" + name, } + metadataList = append(metadataList, metadata) } } - return metadataList, false, nil + +ret: + sort.Sort(ByObjectKey(metadataList)) + return metadataList, resources, nil } func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error { diff --git a/pkg/storage/inmemory/inmemory.go b/pkg/storage/inmemory/inmemory.go index 77de88980..69cad2ef4 100644 --- a/pkg/storage/inmemory/inmemory.go +++ b/pkg/storage/inmemory/inmemory.go @@ -128,31 +128,31 @@ func (storage *storage) StoreBucket(bucketName string) error { return nil } -func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorage.ObjectMetadata, bool, error) { +func (storage *storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) { if _, ok := storage.bucketdata[bucket]; ok == false { - return []mstorage.ObjectMetadata{}, false, mstorage.BucketNotFound{Bucket: bucket} + return []mstorage.ObjectMetadata{}, mstorage.BucketResourcesMetadata{IsTruncated: false}, mstorage.BucketNotFound{Bucket: bucket} } // TODO prefix and count handling var results []mstorage.ObjectMetadata var keys []string for key := range storage.objectdata { - if strings.HasPrefix(key, bucket+":"+prefix) { + if strings.HasPrefix(key, bucket+":"+resources.Prefix) { keys = append(keys, key) } } sort.Strings(keys) for _, key := range keys { - if len(results) == count { - return results, true, nil + if len(results) == resources.Maxkeys { + return results, mstorage.BucketResourcesMetadata{IsTruncated: true}, nil } object := storage.objectdata[key] if bucket == object.metadata.Bucket { - if strings.HasPrefix(key, bucket+":") { + if strings.HasPrefix(key, bucket+":"+resources.Prefix) { results = append(results, object.metadata) } } } - return results, false, nil + return results, resources, nil } type ByBucketName []mstorage.BucketMetadata diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 0d69defde..2abf3a632 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -32,7 +32,7 @@ type Storage interface { // Object Operations CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) GetObjectMetadata(bucket string, object string) (ObjectMetadata, error) - ListObjects(bucket, prefix string, count int) ([]ObjectMetadata, bool, error) + ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error) StoreObject(bucket string, key string, contentType string, data io.Reader) error } @@ -42,8 +42,12 @@ type BucketMetadata struct { } type ObjectMetadata struct { - Bucket string - Key string + Bucket string + Key string + Maxkeys int + Prefix string + Marker string + Delimiter string ContentType string Created time.Time @@ -51,6 +55,21 @@ type ObjectMetadata struct { Size int64 } +// Various types of bucket resources +type BucketResourcesMetadata struct { + Prefix string + Marker string + Maxkeys int + Delimiter string + IsTruncated bool + CommonPrefixes []string + + Policy bool + // TODO + Logging string + Notification string +} + func IsValidBucket(bucket string) bool { if len(bucket) < 3 || len(bucket) > 63 { return false diff --git a/pkg/storage/storage_api_suite.go b/pkg/storage/storage_api_suite.go index 09ddc5537..be38db8c6 100644 --- a/pkg/storage/storage_api_suite.go +++ b/pkg/storage/storage_api_suite.go @@ -81,40 +81,46 @@ func testMultipleObjectCreation(c *C, create func() Storage) { func testPaging(c *C, create func() Storage) { storage := create() storage.StoreBucket("bucket") - storage.ListObjects("bucket", "", 5) - objects, isTruncated, err := storage.ListObjects("bucket", "", 5) + resources := BucketResourcesMetadata{} + objects, resources, err := storage.ListObjects("bucket", resources) c.Assert(len(objects), Equals, 0) - c.Assert(isTruncated, Equals, false) + c.Assert(resources.IsTruncated, Equals, false) c.Assert(err, IsNil) // check before paging occurs for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) storage.StoreObject("bucket", key, "", bytes.NewBufferString(key)) - objects, isTruncated, err = storage.ListObjects("bucket", "", 5) + resources.Maxkeys = 5 + objects, resources, err = storage.ListObjects("bucket", resources) c.Assert(len(objects), Equals, i+1) - c.Assert(isTruncated, Equals, false) + c.Assert(resources.IsTruncated, Equals, false) c.Assert(err, IsNil) } // check after paging occurs pages work for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) storage.StoreObject("bucket", key, "", bytes.NewBufferString(key)) - objects, isTruncated, err = storage.ListObjects("bucket", "", 5) + resources.Maxkeys = 5 + objects, resources, err = storage.ListObjects("bucket", resources) c.Assert(len(objects), Equals, 5) - c.Assert(isTruncated, Equals, true) + c.Assert(resources.IsTruncated, Equals, true) c.Assert(err, IsNil) } // check paging with prefix at end returns less objects { storage.StoreObject("bucket", "newPrefix", "", bytes.NewBufferString("prefix1")) storage.StoreObject("bucket", "newPrefix2", "", bytes.NewBufferString("prefix2")) - objects, isTruncated, err = storage.ListObjects("bucket", "new", 5) + resources.Prefix = "new" + resources.Maxkeys = 5 + objects, resources, err = storage.ListObjects("bucket", resources) c.Assert(len(objects), Equals, 2) } // check ordering of pages { - objects, isTruncated, err = storage.ListObjects("bucket", "", 5) + resources.Prefix = "" + resources.Maxkeys = 1000 + objects, resources, err = storage.ListObjects("bucket", resources) c.Assert(objects[0].Key, Equals, "newPrefix") c.Assert(objects[1].Key, Equals, "newPrefix2") c.Assert(objects[2].Key, Equals, "obj0") @@ -123,7 +129,9 @@ func testPaging(c *C, create func() Storage) { } // check ordering of results with prefix { - objects, isTruncated, err = storage.ListObjects("bucket", "obj", 5) + resources.Prefix = "obj" + resources.Maxkeys = 1000 + objects, resources, err = storage.ListObjects("bucket", resources) c.Assert(objects[0].Key, Equals, "obj0") c.Assert(objects[1].Key, Equals, "obj1") c.Assert(objects[2].Key, Equals, "obj10") @@ -132,7 +140,9 @@ func testPaging(c *C, create func() Storage) { } // check ordering of results with prefix and no paging { - objects, isTruncated, err = storage.ListObjects("bucket", "new", 5) + resources.Prefix = "new" + resources.Maxkeys = 5 + objects, resources, err = storage.ListObjects("bucket", resources) c.Assert(objects[0].Key, Equals, "newPrefix") c.Assert(objects[1].Key, Equals, "newPrefix2") } @@ -230,9 +240,10 @@ func testListBucketsOrder(c *C, create func() Storage) { func testListObjectsTestsForNonExistantBucket(c *C, create func() Storage) { storage := create() - objects, isTruncated, err := storage.ListObjects("bucket", "", 1000) + resources := BucketResourcesMetadata{Prefix: "", Maxkeys: 1000} + objects, resources, err := storage.ListObjects("bucket", resources) c.Assert(err, Not(IsNil)) - c.Assert(isTruncated, Equals, false) + c.Assert(resources.IsTruncated, Equals, false) c.Assert(len(objects), Equals, 0) } diff --git a/pkg/utils/crypto/sha256/sha256.go b/pkg/utils/crypto/sha256/sha256.go index 70f072c91..3f90143a7 100644 --- a/pkg/utils/crypto/sha256/sha256.go +++ b/pkg/utils/crypto/sha256/sha256.go @@ -7,7 +7,7 @@ // Using this part of Minio codebase under the license // Apache License Version 2.0 with modifications -// Package sha256 SHA256SSE3, SHA256AVX, SHA256AVX2 +// Package sha256 provides SHA256SSE3, SHA256AVX, SHA256AVX2 package sha256 import (