Merge pull request #468 from fkautz/pr_out_adding_lru_to_memory_driver_not_wired_to_command_line_opts

Godeps/Godeps.json

@@ -1,6 +1,6 @@
 {
 	"ImportPath": "github.com/minio-io/minio",
-	"GoVersion": "go1.4",
+	"GoVersion": "go1.4.2",
 	"Packages": [
 		"./..."
 	],
@@ -9,6 +9,10 @@
 			"ImportPath": "github.com/dustin/go-humanize",
 			"Rev": "8cc1aaa2d955ee82833337cfb10babc42be6bce6"
 		},
+		{
+			"ImportPath": "github.com/golang/groupcache/lru",
+			"Rev": "604ed5785183e59ae2789449d89e73f3a2a77987"
+		},
 		{
 			"ImportPath": "github.com/gorilla/context",
 			"Rev": "50c25fb3b2b3b3cc724e9b6ac75fb44b3bccd0da"

Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2013 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package lru implements an LRU cache.
+package lru
+
+import "container/list"
+
+// Cache is an LRU cache. It is not safe for concurrent access.
+type Cache struct {
+	// MaxEntries is the maximum number of cache entries before
+	// an item is evicted. Zero means no limit.
+	MaxEntries int
+
+	// OnEvicted optionally specificies a callback function to be
+	// executed when an entry is purged from the cache.
+	OnEvicted func(key Key, value interface{})
+
+	ll    *list.List
+	cache map[interface{}]*list.Element
+}
+
+// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
+type Key interface{}
+
+type entry struct {
+	key   Key
+	value interface{}
+}
+
+// New creates a new Cache.
+// If maxEntries is zero, the cache has no limit and it's assumed
+// that eviction is done by the caller.
+func New(maxEntries int) *Cache {
+	return &Cache{
+		MaxEntries: maxEntries,
+		ll:         list.New(),
+		cache:      make(map[interface{}]*list.Element),
+	}
+}
+
+// Add adds a value to the cache.
+func (c *Cache) Add(key Key, value interface{}) {
+	if c.cache == nil {
+		c.cache = make(map[interface{}]*list.Element)
+		c.ll = list.New()
+	}
+	if ee, ok := c.cache[key]; ok {
+		c.ll.MoveToFront(ee)
+		ee.Value.(*entry).value = value
+		return
+	}
+	ele := c.ll.PushFront(&entry{key, value})
+	c.cache[key] = ele
+	if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
+		c.RemoveOldest()
+	}
+}
+
+// Get looks up a key's value from the cache.
+func (c *Cache) Get(key Key) (value interface{}, ok bool) {
+	if c.cache == nil {
+		return
+	}
+	if ele, hit := c.cache[key]; hit {
+		c.ll.MoveToFront(ele)
+		return ele.Value.(*entry).value, true
+	}
+	return
+}
+
+// Remove removes the provided key from the cache.
+func (c *Cache) Remove(key Key) {
+	if c.cache == nil {
+		return
+	}
+	if ele, hit := c.cache[key]; hit {
+		c.removeElement(ele)
+	}
+}
+
+// RemoveOldest removes the oldest item from the cache.
+func (c *Cache) RemoveOldest() {
+	if c.cache == nil {
+		return
+	}
+	ele := c.ll.Back()
+	if ele != nil {
+		c.removeElement(ele)
+	}
+}
+
+func (c *Cache) removeElement(e *list.Element) {
+	c.ll.Remove(e)
+	kv := e.Value.(*entry)
+	delete(c.cache, kv.key)
+	if c.OnEvicted != nil {
+		c.OnEvicted(kv.key, kv.value)
+	}
+}
+
+// Len returns the number of items in the cache.
+func (c *Cache) Len() int {
+	if c.cache == nil {
+		return 0
+	}
+	return c.ll.Len()
+}

Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2013 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lru
+
+import (
+	"testing"
+)
+
+type simpleStruct struct {
+	int
+	string
+}
+
+type complexStruct struct {
+	int
+	simpleStruct
+}
+
+var getTests = []struct {
+	name       string
+	keyToAdd   interface{}
+	keyToGet   interface{}
+	expectedOk bool
+}{
+	{"string_hit", "myKey", "myKey", true},
+	{"string_miss", "myKey", "nonsense", false},
+	{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
+	{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
+	{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
+		complexStruct{1, simpleStruct{2, "three"}}, true},
+}
+
+func TestGet(t *testing.T) {
+	for _, tt := range getTests {
+		lru := New(0)
+		lru.Add(tt.keyToAdd, 1234)
+		val, ok := lru.Get(tt.keyToGet)
+		if ok != tt.expectedOk {
+			t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
+		} else if ok && val != 1234 {
+			t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
+		}
+	}
+}
+
+func TestRemove(t *testing.T) {
+	lru := New(0)
+	lru.Add("myKey", 1234)
+	if val, ok := lru.Get("myKey"); !ok {
+		t.Fatal("TestRemove returned no match")
+	} else if val != 1234 {
+		t.Fatalf("TestRemove failed.  Expected %d, got %v", 1234, val)
+	}
+
+	lru.Remove("myKey")
+	if _, ok := lru.Get("myKey"); ok {
+		t.Fatal("TestRemove returned a removed entry")
+	}
+}

pkg/api/api_test.go

@@ -60,7 +60,7 @@ var _ = Suite(&MySuite{
 
 var _ = Suite(&MySuite{
 	initDriver: func() (drivers.Driver, string) {
-		_, _, driver := memory.Start()
+		_, _, driver := memory.Start(1000)
 		return driver, ""
 	},
 })

pkg/server/server.go

@@ -136,7 +136,7 @@ func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, status
 	switch {
 	case driverType == Memory:
 		{
-			ctrlChan, statusChan, driver = memory.Start()
+			ctrlChan, statusChan, driver = memory.Start(1000)
 			ctrlChans = append(ctrlChans, ctrlChan)
 			statusChans = append(statusChans, statusChan)
 		}

pkg/storage/drivers/memory/memory.go

@@ -30,14 +30,16 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 
+	"github.com/golang/groupcache/lru"
 	"io/ioutil"
 )
 
 // memoryDriver - local variables
 type memoryDriver struct {
-	bucketdata map[string]storedBucket
+	bucketMetadata map[string]storedBucket
-	objectdata map[string]storedObject
+	objectMetadata map[string]storedObject
-	lock       *sync.RWMutex
+	objects        *lru.Cache
+	lock           *sync.RWMutex
 }
 
 type storedBucket struct {
@@ -48,19 +50,21 @@ type storedBucket struct {
 
 type storedObject struct {
 	metadata drivers.ObjectMetadata
-	data     []byte
 }
 
 // Start memory object server
-func Start() (chan<- string, <-chan error, drivers.Driver) {
+func Start(maxObjects int) (chan<- string, <-chan error, drivers.Driver) {
 	ctrlChannel := make(chan string)
 	errorChannel := make(chan error)
 
 	memory := new(memoryDriver)
-	memory.bucketdata = make(map[string]storedBucket)
+	memory.bucketMetadata = make(map[string]storedBucket)
-	memory.objectdata = make(map[string]storedObject)
+	memory.objectMetadata = make(map[string]storedObject)
+	memory.objects = lru.New(maxObjects)
 	memory.lock = new(sync.RWMutex)
 
+	memory.objects.OnEvicted = memory.evictObject
+
 	go start(ctrlChannel, errorChannel)
 	return ctrlChannel, errorChannel, memory
 }
@@ -73,15 +77,18 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error) {
 func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		return 0, drivers.BucketNotFound{Bucket: bucket}
 	}
 	// get object
-	key := object
+	objectKey := bucket + "/" + object
-	if val, ok := memory.objectdata[key]; ok {
+	if _, ok := memory.objectMetadata[objectKey]; ok {
-		objectBuffer := bytes.NewBuffer(val.data)
+		if data, ok := memory.objects.Get(objectKey); ok {
-		written, err := io.Copy(w, objectBuffer)
+			dataSlice := data.([]byte)
-		return written, err
+			objectBuffer := bytes.NewBuffer(dataSlice)
+			written, err := io.Copy(w, objectBuffer)
+			return written, err
+		}
 	}
 	return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
 }
@@ -104,10 +111,10 @@ func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string,
 func (memory memoryDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		return drivers.BucketMetadata{}, drivers.BucketNotFound{Bucket: bucket}
 	}
-	return memory.bucketdata[bucket].metadata, nil
+	return memory.bucketMetadata[bucket].metadata, nil
 }
 
 // CreateBucketPolicy - Not implemented
@@ -124,12 +131,14 @@ func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy,
 func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
 	memory.lock.RLock()
 
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		memory.lock.RUnlock()
 		return drivers.BucketNotFound{Bucket: bucket}
 	}
 
-	if _, ok := memory.objectdata[key]; ok == true {
+	objectKey := bucket + "/" + key
+
+	if _, ok := memory.objectMetadata[objectKey]; ok == true {
 		memory.lock.RUnlock()
 		return drivers.ObjectExists{Bucket: bucket, Object: key}
 	}
@@ -143,6 +152,7 @@ func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string,
 
 	var bytesBuffer bytes.Buffer
 	var newObject = storedObject{}
+	var dataSlice []byte
 	if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
 		size := bytesBuffer.Len()
 		md5SumBytes := md5.Sum(bytesBuffer.Bytes())
@@ -156,19 +166,15 @@ func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string,
 			Md5:         md5Sum,
 			Size:        int64(size),
 		}
-		newObject.data = bytesBuffer.Bytes()
+		dataSlice = bytesBuffer.Bytes()
 	}
 	memory.lock.Lock()
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.objectMetadata[objectKey]; ok == true {
-		memory.lock.Unlock()
-		return drivers.BucketNotFound{Bucket: bucket}
-	}
-
-	if _, ok := memory.objectdata[key]; ok == true {
 		memory.lock.Unlock()
 		return drivers.ObjectExists{Bucket: bucket, Object: key}
 	}
-	memory.objectdata[key] = newObject
+	memory.objectMetadata[objectKey] = newObject
+	memory.objects.Add(objectKey, dataSlice)
 	memory.lock.Unlock()
 	return nil
 }
@@ -181,7 +187,7 @@ func (memory memoryDriver) CreateBucket(bucketName string) error {
 		return drivers.BucketNameInvalid{Bucket: bucketName}
 	}
 
-	if _, ok := memory.bucketdata[bucketName]; ok == true {
+	if _, ok := memory.bucketMetadata[bucketName]; ok == true {
 		memory.lock.RUnlock()
 		return drivers.BucketExists{Bucket: bucketName}
 	}
@@ -193,7 +199,7 @@ func (memory memoryDriver) CreateBucket(bucketName string) error {
 	newBucket.metadata.Created = time.Now()
 	memory.lock.Lock()
 	defer memory.lock.Unlock()
-	memory.bucketdata[bucketName] = newBucket
+	memory.bucketMetadata[bucketName] = newBucket
 
 	return nil
 }
@@ -233,35 +239,38 @@ func (memory memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedNa
 func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
 	}
 	var results []drivers.ObjectMetadata
 	var keys []string
-	for key := range memory.objectdata {
+	for key := range memory.objectMetadata {
-		switch true {
+		if strings.HasPrefix(key, bucket+"/") {
-		// Prefix absent, delimit object key based on delimiter
+			key = key[len(bucket)+1:]
-		case resources.IsDelimiterSet():
-			delimitedName := delimiter(key, resources.Delimiter)
 			switch true {
-			case delimitedName == "" || delimitedName == key:
+			// Prefix absent, delimit object key based on delimiter
+			case resources.IsDelimiterSet():
+				delimitedName := delimiter(key, resources.Delimiter)
+				switch true {
+				case delimitedName == "" || delimitedName == key:
+					keys = appendUniq(keys, key)
+				case delimitedName != "":
+					resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
+				}
+			// Prefix present, delimit object key with prefix key based on delimiter
+			case resources.IsDelimiterPrefixSet():
+				if strings.HasPrefix(key, resources.Prefix) {
+					trimmedName := strings.TrimPrefix(key, resources.Prefix)
+					delimitedName := delimiter(trimmedName, resources.Delimiter)
+					resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
+				}
+			// Prefix present, nothing to delimit
+			case resources.IsPrefixSet():
+				keys = appendUniq(keys, key)
+			// Prefix and delimiter absent
+			case resources.IsDefault():
 				keys = appendUniq(keys, key)
-			case delimitedName != "":
-				resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
 			}
-		// Prefix present, delimit object key with prefix key based on delimiter
-		case resources.IsDelimiterPrefixSet():
-			if strings.HasPrefix(key, resources.Prefix) {
-				trimmedName := strings.TrimPrefix(key, resources.Prefix)
-				delimitedName := delimiter(trimmedName, resources.Delimiter)
-				resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
-			}
-		// Prefix present, nothing to delimit
-		case resources.IsPrefixSet():
-			keys = appendUniq(keys, key)
-		// Prefix and delimiter absent
-		case resources.IsDefault():
-			keys = appendUniq(keys, key)
 		}
 	}
 	sort.Strings(keys)
@@ -269,7 +278,7 @@ func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketRe
 		if len(results) == resources.Maxkeys {
 			return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil
 		}
-		object := memory.objectdata[key]
+		object := memory.objectMetadata[bucket+"/"+key]
 		if bucket == object.metadata.Bucket {
 			results = append(results, object.metadata)
 		}
@@ -294,7 +303,7 @@ func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
 	var results []drivers.BucketMetadata
-	for _, bucket := range memory.bucketdata {
+	for _, bucket := range memory.bucketMetadata {
 		results = append(results, bucket.metadata)
 	}
 	sort.Sort(ByBucketName(results))
@@ -306,11 +315,20 @@ func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (driver
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
 	// check if bucket exists
-	if _, ok := memory.bucketdata[bucket]; ok == false {
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket}
 	}
-	if object, ok := memory.objectdata[key]; ok == true {
+
+	objectKey := bucket + "/" + key
+
+	if object, ok := memory.objectMetadata[objectKey]; ok == true {
 		return object.metadata, nil
 	}
 	return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
 }
+
+func (memory memoryDriver) evictObject(key lru.Key, value interface{}) {
+	k := key.(string)
+
+	delete(memory.objectMetadata, k)
+}

pkg/storage/drivers/memory/memory_test.go

@@ -31,7 +31,7 @@ var _ = Suite(&MySuite{})
 
 func (s *MySuite) TestAPISuite(c *C) {
 	create := func() drivers.Driver {
-		_, _, store := Start()
+		_, _, store := Start(1000)
 		return store
 	}
 	drivers.APITestSuite(c, create)