Add lexicographic Marker/NextMarker support for recursive listing of objects.

Also update times when an object is accessed logic

pkg/storage/donut/objectstorage.go

@@ -92,7 +92,6 @@ func (d donut) ListBuckets() (results []string, err error) {
 
 // ListObjects - return list of objects
 func (d donut) ListObjects(bucket, prefix, marker, delimiter string, maxkeys int) ([]string, []string, bool, error) {
-	// TODO: Marker is not yet handled please handle it
 	errParams := map[string]string{
 		"bucket":    bucket,
 		"prefix":    prefix,

pkg/storage/drivers/donut/donut.go

@@ -329,17 +329,16 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso
 	if !drivers.IsValidObjectName(resources.Prefix) {
 		return nil, drivers.BucketResourcesMetadata{}, iodine.New(drivers.ObjectNameInvalid{Object: resources.Prefix}, nil)
 	}
-	actualObjects, commonPrefixes, isTruncated, err := d.donut.ListObjects(bucketName,
-		resources.Prefix,
-		resources.Marker,
-		resources.Delimiter,
+	actualObjects, commonPrefixes, isTruncated, err := d.donut.ListObjects(bucketName, resources.Prefix, resources.Marker, resources.Delimiter,
 		resources.Maxkeys)
 	if err != nil {
 		return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams)
 	}
 	resources.CommonPrefixes = commonPrefixes
 	resources.IsTruncated = isTruncated
-
+	if resources.IsTruncated && resources.IsDelimiterSet() {
+		resources.NextMarker = actualObjects[len(actualObjects)-1]
+	}
 	var results []drivers.ObjectMetadata
 	for _, objectName := range actualObjects {
 		objectMetadata, err := d.donut.GetObjectMetadata(bucketName, objectName)

pkg/storage/drivers/driver.go

@@ -102,7 +102,9 @@ const (
 type BucketResourcesMetadata struct {
 	Prefix         string
 	Marker         string
+	NextMarker     string
 	Maxkeys        int
+	EncodingType   string
 	Delimiter      string
 	IsTruncated    bool
 	CommonPrefixes []string

pkg/storage/drivers/memory/memory.go

@@ -66,7 +66,7 @@ func Start(maxSize uint64, expiration time.Duration) (chan<- string, <-chan erro
 	memory.objects.OnEvicted = memory.evictObject
 
 	// set up memory expiration
-	memory.objects.ExpireObjects(time.Millisecond * 10)
+	memory.objects.ExpireObjects(time.Second * 5)
 
 	go start(ctrlChannel, errorChannel)
 	return ctrlChannel, errorChannel, memory
@@ -356,45 +356,45 @@ func appendUniq(slice []string, i string) []string {
 	return append(slice, i)
 }
 
-func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedName string, resources drivers.BucketResourcesMetadata) (drivers.BucketResourcesMetadata, []string) {
+func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delim string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) {
 	switch true {
-	case key == resources.Prefix:
+	case key == r.Prefix:
 		keys = appendUniq(keys, key)
-	// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
-	case key == resources.Prefix+delimitedName:
+	// delim - requires r.Prefix as it was trimmed off earlier
+	case key == r.Prefix+delim:
 		keys = appendUniq(keys, key)
-	case delimitedName != "":
-		resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
+	case delim != "":
+		r.CommonPrefixes = appendUniq(r.CommonPrefixes, r.Prefix+delim)
 	}
-	return resources, keys
+	return keys, r
 }
 
-func (memory *memoryDriver) listObjectsInternal(keys []string, key string, resources drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) {
+func (memory *memoryDriver) listObjects(keys []string, key string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) {
 	switch true {
 	// Prefix absent, delimit object key based on delimiter
-	case resources.IsDelimiterSet():
-		delimitedName := delimiter(key, resources.Delimiter)
+	case r.IsDelimiterSet():
+		delim := delimiter(key, r.Delimiter)
 		switch true {
-		case delimitedName == "" || delimitedName == key:
+		case delim == "" || delim == key:
 			keys = appendUniq(keys, key)
-		case delimitedName != "":
-			resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
+		case delim != "":
+			r.CommonPrefixes = appendUniq(r.CommonPrefixes, delim)
 		}
 	// Prefix present, delimit object key with prefix key based on delimiter
-	case resources.IsDelimiterPrefixSet():
-		if strings.HasPrefix(key, resources.Prefix) {
-			trimmedName := strings.TrimPrefix(key, resources.Prefix)
-			delimitedName := delimiter(trimmedName, resources.Delimiter)
-			resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
+	case r.IsDelimiterPrefixSet():
+		if strings.HasPrefix(key, r.Prefix) {
+			trimmedName := strings.TrimPrefix(key, r.Prefix)
+			delim := delimiter(trimmedName, r.Delimiter)
+			keys, r = memory.filterDelimiterPrefix(keys, key, delim, r)
 		}
 	// Prefix present, nothing to delimit
-	case resources.IsPrefixSet():
+	case r.IsPrefixSet():
 		keys = appendUniq(keys, key)
 	// Prefix and delimiter absent
-	case resources.IsDefault():
+	case r.IsDefault():
 		keys = appendUniq(keys, key)
 	}
-	return keys, resources
+	return keys, r
 }
 
 // ListObjects - list objects from memory
@@ -416,13 +416,28 @@ func (memory *memoryDriver) ListObjects(bucket string, resources drivers.BucketR
 	for key := range storedBucket.objectMetadata {
 		if strings.HasPrefix(key, bucket+"/") {
 			key = key[len(bucket)+1:]
-			keys, resources = memory.listObjectsInternal(keys, key, resources)
+			keys, resources = memory.listObjects(keys, key, resources)
 		}
 	}
-	sort.Strings(keys)
-	for _, key := range keys {
+	// Marker logic - TODO in-efficient right now fix it
+	var newKeys []string
+	switch {
+	case resources.Marker != "":
+		for _, key := range keys {
+			if key > resources.Marker {
+				newKeys = appendUniq(newKeys, key)
+			}
+		}
+	default:
+		newKeys = keys
+	}
+	sort.Strings(newKeys)
+	for _, key := range newKeys {
 		if len(results) == resources.Maxkeys {
 			resources.IsTruncated = true
+			if resources.IsTruncated && resources.IsDelimiterSet() {
+				resources.NextMarker = results[len(results)-1].Key
+			}
 			return results, resources, nil
 		}
 		object := storedBucket.objectMetadata[bucket+"/"+key]

pkg/storage/drivers/memory/memory_intelligent.go

@@ -33,8 +33,8 @@ type Intelligent struct {
 	// items hold the cached objects
 	items map[string]interface{}
 
-	// createdAt holds the time that related item's created At
-	createdAt map[string]time.Time
+	// updatedAt holds the time that related item's updated at
+	updatedAt map[string]time.Time
 
 	// expiration is a duration for a cache key to expire
 	expiration time.Duration
@@ -69,7 +69,7 @@ type Stats struct {
 func NewIntelligent(maxSize uint64, expiration time.Duration) *Intelligent {
 	return &Intelligent{
 		items:      map[string]interface{}{},
-		createdAt:  map[string]time.Time{},
+		updatedAt:  map[string]time.Time{},
 		expiration: expiration,
 		maxSize:    maxSize,
 	}
@@ -91,7 +91,6 @@ func (r *Intelligent) ExpireObjects(gcInterval time.Duration) {
 		for range time.Tick(gcInterval) {
 			r.Lock()
 			for key := range r.items {
-
 				if !r.isValid(key) {
 					r.Delete(key)
 				}
@@ -106,7 +105,11 @@ func (r *Intelligent) Get(key string) (interface{}, bool) {
 	r.Lock()
 	defer r.Unlock()
 	value, ok := r.items[key]
-	return value, ok
+	if !ok {
+		return nil, false
+	}
+	r.updatedAt[key] = time.Now()
+	return value, true
 }
 
 // Set will persist a value to the cache
@@ -124,7 +127,7 @@ func (r *Intelligent) Set(key string, value interface{}) {
 	}
 	r.items[key] = value
 	r.currentSize += uint64(len(value.([]byte)))
-	r.createdAt[key] = time.Now()
+	r.updatedAt[key] = time.Now()
 	r.Unlock()
 	return
 }
@@ -133,7 +136,7 @@ func (r *Intelligent) Set(key string, value interface{}) {
 func (r *Intelligent) Delete(key string) {
 	r.currentSize -= uint64(len(r.items[key].([]byte)))
 	delete(r.items, key)
-	delete(r.createdAt, key)
+	delete(r.updatedAt, key)
 	r.totalEvicted++
 	if r.OnEvicted != nil {
 		r.OnEvicted(key)
@@ -141,12 +144,12 @@ func (r *Intelligent) Delete(key string) {
 }
 
 func (r *Intelligent) isValid(key string) bool {
-	createdAt, ok := r.createdAt[key]
+	updatedAt, ok := r.updatedAt[key]
 	if !ok {
 		return false
 	}
 	if r.expiration == zeroExpiration {
 		return true
 	}
-	return createdAt.Add(r.expiration).After(time.Now())
+	return updatedAt.Add(r.expiration).After(time.Now())
 }