diff --git a/pkg/storage/donut/bucket.go b/pkg/storage/donut/bucket.go index 1c798ffa9..6e4012902 100644 --- a/pkg/storage/donut/bucket.go +++ b/pkg/storage/donut/bucket.go @@ -145,7 +145,7 @@ func (b bucket) ListObjects(prefix, marker, delimiter string, maxkeys int) (List for objectName := range bucketMetadata.Buckets[b.getBucketName()].BucketObjects { if strings.HasPrefix(objectName, strings.TrimSpace(prefix)) { if objectName > marker { - objects = AppendU(objects, objectName) + objects = append(objects, objectName) } } } @@ -166,16 +166,19 @@ func (b bucket) ListObjects(prefix, marker, delimiter string, maxkeys int) (List var commonPrefixes []string for _, commonPrefix := range prefixes { - commonPrefixes = AppendU(commonPrefixes, prefix+commonPrefix) + commonPrefixes = append(commonPrefixes, prefix+commonPrefix) } + filteredObjects = RemoveDuplicates(filteredObjects) sort.Strings(filteredObjects) for _, objectName := range filteredObjects { if len(results) >= maxkeys { isTruncated = true break } - results = AppendU(results, prefix+objectName) + results = append(results, prefix+objectName) } + results = RemoveDuplicates(results) + commonPrefixes = RemoveDuplicates(commonPrefixes) sort.Strings(commonPrefixes) listObjects := ListObjects{} diff --git a/pkg/storage/donut/cache.go b/pkg/storage/donut/cache.go index ad8ab4929..3c089cd5b 100644 --- a/pkg/storage/donut/cache.go +++ b/pkg/storage/donut/cache.go @@ -17,7 +17,6 @@ package donut import ( - "bufio" "bytes" "crypto/md5" "encoding/base64" @@ -419,63 +418,47 @@ func (cache Cache) CreateBucket(bucketName, acl string) error { return nil } -func delimiter(object, delimiter string) string { - readBuffer := bytes.NewBufferString(object) - reader := bufio.NewReader(readBuffer) - stringReader := strings.NewReader(delimiter) - delimited, _ := stringReader.ReadByte() - delimitedStr, _ := reader.ReadString(delimited) - return delimitedStr -} - -func appendUniq(slice []string, i string) []string { - for _, ele := range slice { - if ele == i { - return slice - } - } - return append(slice, i) -} - -func (cache Cache) filterDelimiterPrefix(keys []string, key, delim string, r BucketResourcesMetadata) ([]string, BucketResourcesMetadata) { +func (cache Cache) filterDelimiterPrefix(keys []string, key, prefix, delim string) ([]string, []string) { + var commonPrefixes []string switch true { - case key == r.Prefix: - keys = appendUniq(keys, key) + case key == prefix: + keys = append(keys, key) // delim - requires r.Prefix as it was trimmed off earlier - case key == r.Prefix+delim: - keys = appendUniq(keys, key) + case key == prefix+delim: + keys = append(keys, key) case delim != "": - r.CommonPrefixes = appendUniq(r.CommonPrefixes, r.Prefix+delim) + commonPrefixes = append(commonPrefixes, prefix+delim) } - return keys, r + return RemoveDuplicates(keys), RemoveDuplicates(commonPrefixes) } -func (cache Cache) listObjects(keys []string, key string, r BucketResourcesMetadata) ([]string, BucketResourcesMetadata) { +func (cache Cache) listObjects(keys []string, key string, r BucketResourcesMetadata) ([]string, []string) { + var commonPrefixes []string switch true { // Prefix absent, delimit object key based on delimiter case r.IsDelimiterSet(): - delim := delimiter(key, r.Delimiter) + delim := Delimiter(key, r.Delimiter) switch true { case delim == "" || delim == key: - keys = appendUniq(keys, key) + keys = append(keys, key) case delim != "": - r.CommonPrefixes = appendUniq(r.CommonPrefixes, delim) + commonPrefixes = append(commonPrefixes, delim) } // Prefix present, delimit object key with prefix key based on delimiter case r.IsDelimiterPrefixSet(): if strings.HasPrefix(key, r.Prefix) { trimmedName := strings.TrimPrefix(key, r.Prefix) - delim := delimiter(trimmedName, r.Delimiter) - keys, r = cache.filterDelimiterPrefix(keys, key, delim, r) + delim := Delimiter(trimmedName, r.Delimiter) + keys, commonPrefixes = cache.filterDelimiterPrefix(keys, key, r.Prefix, delim) } // Prefix present, nothing to delimit case r.IsPrefixSet(): - keys = appendUniq(keys, key) + keys = append(keys, key) // Prefix and delimiter absent case r.IsDefault(): - keys = appendUniq(keys, key) + keys = append(keys, key) } - return keys, r + return RemoveDuplicates(keys), RemoveDuplicates(commonPrefixes) } // ListObjects - list objects from cache @@ -493,11 +476,12 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) } var results []ObjectMetadata var keys []string + var commonPrefixes []string storedBucket := cache.storedBuckets[bucket] for key := range storedBucket.objectMetadata { if strings.HasPrefix(key, bucket+"/") { key = key[len(bucket)+1:] - keys, resources = cache.listObjects(keys, key, resources) + keys, commonPrefixes = cache.listObjects(keys, key, resources) } } var newKeys []string @@ -505,12 +489,13 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) case resources.Marker != "": for _, key := range keys { if key > resources.Marker { - newKeys = appendUniq(newKeys, key) + newKeys = append(newKeys, key) } } default: newKeys = keys } + newKeys = RemoveDuplicates(newKeys) sort.Strings(newKeys) for _, key := range newKeys { if len(results) == resources.Maxkeys { @@ -523,6 +508,7 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) object := storedBucket.objectMetadata[bucket+"/"+key] results = append(results, object) } + resources.CommonPrefixes = commonPrefixes return results, resources, nil } diff --git a/pkg/storage/donut/common.go b/pkg/storage/donut/common.go index 9fbb18eb6..e55fc3739 100644 --- a/pkg/storage/donut/common.go +++ b/pkg/storage/donut/common.go @@ -17,18 +17,33 @@ package donut import ( + "bufio" + "bytes" "sort" "strings" ) -// AppendU append to an input slice if the element is unique and provides a new slice -func AppendU(slice []string, i string) []string { - for _, ele := range slice { - if ele == i { - return slice +// Delimiter delims the string at delimiter +func Delimiter(object, delimiter string) string { + readBuffer := bytes.NewBufferString(object) + reader := bufio.NewReader(readBuffer) + stringReader := strings.NewReader(delimiter) + delimited, _ := stringReader.ReadByte() + delimitedStr, _ := reader.ReadString(delimited) + return delimitedStr +} + +// RemoveDuplicates removes duplicate elements from a slice +func RemoveDuplicates(slice []string) []string { + newSlice := []string{} + seen := make(map[string]struct{}) + for _, val := range slice { + if _, ok := seen[val]; !ok { + newSlice = append(newSlice, val) + seen[val] = struct{}{} // avoiding byte allocation } } - return append(slice, i) + return newSlice } // TrimPrefix trims off a prefix string from all the elements in a given slice diff --git a/pkg/storage/donut/donut_test.go b/pkg/storage/donut/donut_test.go index d1dad9c45..fad0a26fa 100644 --- a/pkg/storage/donut/donut_test.go +++ b/pkg/storage/donut/donut_test.go @@ -94,7 +94,7 @@ func (s *MySuite) TestEmptyBucket(c *C) { listObjects, err := donut.ListObjects("foo", "", "", "", 1) c.Assert(err, IsNil) c.Assert(len(listObjects.Objects), Equals, 0) - c.Assert(listObjects.CommonPrefixes, IsNil) + c.Assert(listObjects.CommonPrefixes, DeepEquals, []string{}) c.Assert(listObjects.IsTruncated, Equals, false) }