Adding lru to memory driver, not wired to command line opts

pkg/api/api_test.go

@@ -60,7 +60,7 @@ var _ = Suite(&MySuite{
 
 var _ = Suite(&MySuite{
 	initDriver: func() (drivers.Driver, string) {
-		_, _, driver := memory.Start()
+		_, _, driver := memory.Start(1000)
 		return driver, ""
 	},
 })

pkg/server/server.go

@@ -136,7 +136,7 @@ func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, status
 	switch {
 	case driverType == Memory:
 		{
-			ctrlChan, statusChan, driver = memory.Start()
+			ctrlChan, statusChan, driver = memory.Start(1000)
 			ctrlChans = append(ctrlChans, ctrlChan)
 			statusChans = append(statusChans, statusChan)
 		}

pkg/storage/drivers/memory/memory.go

@@ -30,14 +30,16 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 
+	"github.com/golang/groupcache/lru"
 	"io/ioutil"
 )
 
 // memoryDriver - local variables
 type memoryDriver struct {
-	bucketdata map[string]storedBucket
-	objectdata map[string]storedObject
-	lock       *sync.RWMutex
+	bucketMetadata map[string]storedBucket
+	objectMetadata map[string]storedObject
+	objects        *lru.Cache
+	lock           *sync.RWMutex
 }
 
 type storedBucket struct {
@@ -48,19 +50,21 @@ type storedBucket struct {
 
 type storedObject struct {
 	metadata drivers.ObjectMetadata
-	data     []byte
 }
 
 // Start memory object server
-func Start() (chan<- string, <-chan error, drivers.Driver) {
+func Start(maxObjects int) (chan<- string, <-chan error, drivers.Driver) {
 	ctrlChannel := make(chan string)
 	errorChannel := make(chan error)
 
 	memory := new(memoryDriver)
-	memory.bucketdata = make(map[string]storedBucket)
-	memory.objectdata = make(map[string]storedObject)
+	memory.bucketMetadata = make(map[string]storedBucket)
+	memory.objectMetadata = make(map[string]storedObject)
+	memory.objects = lru.New(maxObjects)
 	memory.lock = new(sync.RWMutex)
 
+	memory.objects.OnEvicted = memory.evictObject
+
 	go start(ctrlChannel, errorChannel)
 	return ctrlChannel, errorChannel, memory
 }
@@ -73,15 +77,18 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error) {
 func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		return 0, drivers.BucketNotFound{Bucket: bucket}
 	}
 	// get object
-	key := object
-	if val, ok := memory.objectdata[key]; ok {
-		objectBuffer := bytes.NewBuffer(val.data)
-		written, err := io.Copy(w, objectBuffer)
-		return written, err
+	objectKey := bucket + "/" + object
+	if _, ok := memory.objectMetadata[objectKey]; ok {
+		if data, ok := memory.objects.Get(objectKey); ok {
+			dataSlice := data.([]byte)
+			objectBuffer := bytes.NewBuffer(dataSlice)
+			written, err := io.Copy(w, objectBuffer)
+			return written, err
+		}
 	}
 	return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
 }
@@ -104,10 +111,10 @@ func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string,
 func (memory memoryDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		return drivers.BucketMetadata{}, drivers.BucketNotFound{Bucket: bucket}
 	}
-	return memory.bucketdata[bucket].metadata, nil
+	return memory.bucketMetadata[bucket].metadata, nil
 }
 
 // CreateBucketPolicy - Not implemented
@@ -124,12 +131,14 @@ func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy,
 func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
 	memory.lock.RLock()
 
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		memory.lock.RUnlock()
 		return drivers.BucketNotFound{Bucket: bucket}
 	}
 
-	if _, ok := memory.objectdata[key]; ok == true {
+	objectKey := bucket + "/" + key
+
+	if _, ok := memory.objectMetadata[objectKey]; ok == true {
 		memory.lock.RUnlock()
 		return drivers.ObjectExists{Bucket: bucket, Object: key}
 	}
@@ -143,6 +152,7 @@ func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string,
 
 	var bytesBuffer bytes.Buffer
 	var newObject = storedObject{}
+	var dataSlice []byte
 	if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
 		size := bytesBuffer.Len()
 		md5SumBytes := md5.Sum(bytesBuffer.Bytes())
@@ -156,19 +166,15 @@ func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string,
 			Md5:         md5Sum,
 			Size:        int64(size),
 		}
-		newObject.data = bytesBuffer.Bytes()
+		dataSlice = bytesBuffer.Bytes()
 	}
 	memory.lock.Lock()
-	if _, ok := memory.bucketdata[bucket]; ok == false {
-		memory.lock.Unlock()
-		return drivers.BucketNotFound{Bucket: bucket}
-	}
-
-	if _, ok := memory.objectdata[key]; ok == true {
+	if _, ok := memory.objectMetadata[objectKey]; ok == true {
 		memory.lock.Unlock()
 		return drivers.ObjectExists{Bucket: bucket, Object: key}
 	}
-	memory.objectdata[key] = newObject
+	memory.objectMetadata[objectKey] = newObject
+	memory.objects.Add(objectKey, dataSlice)
 	memory.lock.Unlock()
 	return nil
 }
@@ -181,7 +187,7 @@ func (memory memoryDriver) CreateBucket(bucketName string) error {
 		return drivers.BucketNameInvalid{Bucket: bucketName}
 	}
 
-	if _, ok := memory.bucketdata[bucketName]; ok == true {
+	if _, ok := memory.bucketMetadata[bucketName]; ok == true {
 		memory.lock.RUnlock()
 		return drivers.BucketExists{Bucket: bucketName}
 	}
@@ -193,7 +199,7 @@ func (memory memoryDriver) CreateBucket(bucketName string) error {
 	newBucket.metadata.Created = time.Now()
 	memory.lock.Lock()
 	defer memory.lock.Unlock()
-	memory.bucketdata[bucketName] = newBucket
+	memory.bucketMetadata[bucketName] = newBucket
 
 	return nil
 }
@@ -233,35 +239,38 @@ func (memory memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedNa
 func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
 	}
 	var results []drivers.ObjectMetadata
 	var keys []string
-	for key := range memory.objectdata {
-		switch true {
-		// Prefix absent, delimit object key based on delimiter
-		case resources.IsDelimiterSet():
-			delimitedName := delimiter(key, resources.Delimiter)
+	for key := range memory.objectMetadata {
+		if strings.HasPrefix(key, bucket+"/") {
+			key = key[len(bucket)+1:]
 			switch true {
-			case delimitedName == "" || delimitedName == key:
+			// Prefix absent, delimit object key based on delimiter
+			case resources.IsDelimiterSet():
+				delimitedName := delimiter(key, resources.Delimiter)
+				switch true {
+				case delimitedName == "" || delimitedName == key:
+					keys = appendUniq(keys, key)
+				case delimitedName != "":
+					resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
+				}
+			// Prefix present, delimit object key with prefix key based on delimiter
+			case resources.IsDelimiterPrefixSet():
+				if strings.HasPrefix(key, resources.Prefix) {
+					trimmedName := strings.TrimPrefix(key, resources.Prefix)
+					delimitedName := delimiter(trimmedName, resources.Delimiter)
+					resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
+				}
+			// Prefix present, nothing to delimit
+			case resources.IsPrefixSet():
+				keys = appendUniq(keys, key)
+			// Prefix and delimiter absent
+			case resources.IsDefault():
 				keys = appendUniq(keys, key)
-			case delimitedName != "":
-				resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
 			}
-		// Prefix present, delimit object key with prefix key based on delimiter
-		case resources.IsDelimiterPrefixSet():
-			if strings.HasPrefix(key, resources.Prefix) {
-				trimmedName := strings.TrimPrefix(key, resources.Prefix)
-				delimitedName := delimiter(trimmedName, resources.Delimiter)
-				resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
-			}
-		// Prefix present, nothing to delimit
-		case resources.IsPrefixSet():
-			keys = appendUniq(keys, key)
-		// Prefix and delimiter absent
-		case resources.IsDefault():
-			keys = appendUniq(keys, key)
 		}
 	}
 	sort.Strings(keys)
@@ -269,7 +278,7 @@ func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketRe
 		if len(results) == resources.Maxkeys {
 			return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil
 		}
-		object := memory.objectdata[key]
+		object := memory.objectMetadata[bucket+"/"+key]
 		if bucket == object.metadata.Bucket {
 			results = append(results, object.metadata)
 		}
@@ -294,7 +303,7 @@ func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
 	var results []drivers.BucketMetadata
-	for _, bucket := range memory.bucketdata {
+	for _, bucket := range memory.bucketMetadata {
 		results = append(results, bucket.metadata)
 	}
 	sort.Sort(ByBucketName(results))
@@ -306,11 +315,20 @@ func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (driver
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
 	// check if bucket exists
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket}
 	}
-	if object, ok := memory.objectdata[key]; ok == true {
+
+	objectKey := bucket + "/" + key
+
+	if object, ok := memory.objectMetadata[objectKey]; ok == true {
 		return object.metadata, nil
 	}
 	return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
 }
+
+func (memory memoryDriver) evictObject(key lru.Key, value interface{}) {
+	k := key.(string)
+
+	delete(memory.objectMetadata, k)
+}

pkg/storage/drivers/memory/memory_test.go

@@ -31,7 +31,7 @@ var _ = Suite(&MySuite{})
 
 func (s *MySuite) TestAPISuite(c *C) {
 	create := func() drivers.Driver {
-		_, _, store := Start()
+		_, _, store := Start(1000)
 		return store
 	}
 	drivers.APITestSuite(c, create)