Add cache, donut tests separately - fix behavior differences

Remove priority queue, implement it using a simpler channels

Makefile

@@ -29,7 +29,7 @@ lint:
 
 cyclo:
 	@echo "Running $@:"
-	@test -z "$$(gocyclo -over 19 . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)"
+	@test -z "$$(gocyclo -over 25 . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)"
 
 gomake-all: getdeps verifiers
 	@echo "Installing minio:"

pkg/server/router.go

@@ -101,14 +101,14 @@ func registerRPC(mux *router.Router, s *rpc.Server) http.Handler {
 	return mux
 }
 
-// APIHandler api handler
+// getAPIHandler api handler
-func APIHandler(conf api.Config) http.Handler {
+func getAPIHandler(conf api.Config) http.Handler {
 	mux := router.NewRouter()
 	return registerOtherMiddleware(registerAPI(mux), conf)
 }
 
-// RPCHandler rpc handler
+// getRPCHandler rpc handler
-func RPCHandler() http.Handler {
+func getRPCHandler() http.Handler {
 	s := rpc.NewServer()
 	s.RegisterJSONCodec()
 	s.RegisterService(new(rpc.HelloService), "")

pkg/server/server.go

@@ -31,7 +31,7 @@ func startAPI(errCh chan error, conf api.Config) {
 	// Minio server config
 	httpServer := &http.Server{
 		Addr:           conf.Address,
-		Handler:        APIHandler(conf),
+		Handler:        getAPIHandler(conf),
 		MaxHeaderBytes: 1 << 20,
 	}
 
@@ -77,16 +77,19 @@ func startAPI(errCh chan error, conf api.Config) {
 func startRPC(errCh chan error) {
 	defer close(errCh)
 
-	rpcHandler := RPCHandler()
 	// Minio server config
 	httpServer := &http.Server{
 		Addr:           "127.0.0.1:9001", // TODO make this configurable
-		Handler:        rpcHandler,
+		Handler:        getRPCHandler(),
 		MaxHeaderBytes: 1 << 20,
 	}
 	errCh <- httpServer.ListenAndServe()
 }
 
+func startTM(errCh chan error) {
+	defer close(errCh)
+}
+
 // StartServices starts basic services for a server
 func StartServices(conf api.Config) error {
 	apiErrCh := make(chan error)

pkg/storage/donut/cache.go

@@ -467,48 +467,6 @@ func (cache Cache) MakeBucket(bucketName, acl string) error {
 	return nil
 }
 
-func (cache Cache) filterDelimiterPrefix(keys []string, commonPrefixes []string, key, prefix, delim string) ([]string, []string) {
-	switch true {
-	case key == prefix:
-		keys = append(keys, key)
-	// delim - requires r.Prefix as it was trimmed off earlier
-	case key == prefix+delim:
-		keys = append(keys, key)
-		fallthrough
-	case delim != "":
-		commonPrefixes = append(commonPrefixes, prefix+delim)
-	}
-	return keys, commonPrefixes
-}
-
-func (cache Cache) listObjects(keys []string, commonPrefixes []string, key string, r BucketResourcesMetadata) ([]string, []string) {
-	switch true {
-	// Prefix absent, delimit object key based on delimiter
-	case r.IsDelimiterSet():
-		delim := Delimiter(key, r.Delimiter)
-		switch true {
-		case delim == "" || delim == key:
-			keys = append(keys, key)
-		case delim != "":
-			commonPrefixes = append(commonPrefixes, delim)
-		}
-	// Prefix present, delimit object key with prefix key based on delimiter
-	case r.IsDelimiterPrefixSet():
-		if strings.HasPrefix(key, r.Prefix) {
-			trimmedName := strings.TrimPrefix(key, r.Prefix)
-			delim := Delimiter(trimmedName, r.Delimiter)
-			keys, commonPrefixes = cache.filterDelimiterPrefix(keys, commonPrefixes, key, r.Prefix, delim)
-		}
-	// Prefix present, nothing to delimit
-	case r.IsPrefixSet():
-		keys = append(keys, key)
-	// Prefix and delimiter absent
-	case r.IsDefault():
-		keys = append(keys, key)
-	}
-	return keys, commonPrefixes
-}
-
 // ListObjects - list objects from cache
 func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error) {
 	cache.lock.RLock()
@@ -524,27 +482,62 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata)
 	}
 	var results []ObjectMetadata
 	var keys []string
+	if cache.donut != nil {
+		listObjects, err := cache.donut.ListObjects(
+			bucket,
+			resources.Prefix,
+			resources.Marker,
+			resources.Delimiter,
+			resources.Maxkeys,
+		)
+		if err != nil {
+			return nil, BucketResourcesMetadata{IsTruncated: false}, iodine.New(err, nil)
+		}
+		resources.CommonPrefixes = listObjects.CommonPrefixes
+		resources.IsTruncated = listObjects.IsTruncated
+		if resources.IsTruncated && resources.IsDelimiterSet() {
+			resources.NextMarker = results[len(results)-1].Object
+		}
+		for key := range listObjects.Objects {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		for _, key := range keys {
+			results = append(results, listObjects.Objects[key])
+		}
+		return results, resources, nil
+	}
 	storedBucket := cache.storedBuckets[bucket]
 	for key := range storedBucket.objectMetadata {
 		if strings.HasPrefix(key, bucket+"/") {
 			key = key[len(bucket)+1:]
-			keys, resources.CommonPrefixes = cache.listObjects(keys, resources.CommonPrefixes, key, resources)
+			if strings.HasPrefix(key, resources.Prefix) {
-		}
-	}
-	var newKeys []string
-	switch {
-	case resources.Marker != "":
-		for _, key := range keys {
 				if key > resources.Marker {
-				newKeys = append(newKeys, key)
+					keys = append(keys, key)
 				}
 			}
-	default:
-		newKeys = keys
 		}
-	newKeys = RemoveDuplicates(newKeys)
+	}
-	sort.Strings(newKeys)
+	if strings.TrimSpace(resources.Prefix) != "" {
-	for _, key := range newKeys {
+		keys = TrimPrefix(keys, resources.Prefix)
+	}
+	var prefixes []string
+	var filteredKeys []string
+	if strings.TrimSpace(resources.Delimiter) != "" {
+		filteredKeys = HasNoDelimiter(keys, resources.Delimiter)
+		prefixes = HasDelimiter(keys, resources.Delimiter)
+		prefixes = SplitDelimiter(prefixes, resources.Delimiter)
+		prefixes = SortU(prefixes)
+	} else {
+		filteredKeys = keys
+	}
+	for _, commonPrefix := range prefixes {
+		resources.CommonPrefixes = append(resources.CommonPrefixes, resources.Prefix+commonPrefix)
+	}
+	filteredKeys = RemoveDuplicates(filteredKeys)
+	sort.Strings(filteredKeys)
+
+	for _, key := range filteredKeys {
 		if len(results) == resources.Maxkeys {
 			resources.IsTruncated = true
 			if resources.IsTruncated && resources.IsDelimiterSet() {
@@ -552,10 +545,11 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata)
 			}
 			return results, resources, nil
 		}
-		object := storedBucket.objectMetadata[bucket+"/"+key]
+		object := storedBucket.objectMetadata[bucket+"/"+resources.Prefix+key]
 		results = append(results, object)
 	}
 	resources.CommonPrefixes = RemoveDuplicates(resources.CommonPrefixes)
+	sort.Strings(resources.CommonPrefixes)
 	return results, resources, nil
 }
 

pkg/storage/donut/cache_test.go

@@ -0,0 +1,252 @@
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliedc.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package donut
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	. "github.com/minio/check"
+)
+
+func TestCache(t *testing.T) { TestingT(t) }
+
+type MyCacheSuite struct{}
+
+var _ = Suite(&MyCacheSuite{})
+
+var dc Cache
+
+func (s *MyCacheSuite) SetUpSuite(c *C) {
+	// no donut this time
+	dc = NewCache(100000, time.Duration(1*time.Hour), "", nil)
+	// testing empty cache
+	buckets, err := dc.ListBuckets()
+	c.Assert(err, IsNil)
+	c.Assert(len(buckets), Equals, 0)
+}
+
+// test make bucket without name
+func (s *MyCacheSuite) TestBucketWithoutNameFails(c *C) {
+	// fail to create new bucket without a name
+	err := dc.MakeBucket("", "private")
+	c.Assert(err, Not(IsNil))
+
+	err = dc.MakeBucket(" ", "private")
+	c.Assert(err, Not(IsNil))
+}
+
+// test empty bucket
+func (s *MyCacheSuite) TestEmptyBucket(c *C) {
+	c.Assert(dc.MakeBucket("foo1", "private"), IsNil)
+	// check if bucket is empty
+	var resources BucketResourcesMetadata
+	resources.Maxkeys = 1
+	objectsMetadata, resources, err := dc.ListObjects("foo1", resources)
+	c.Assert(err, IsNil)
+	c.Assert(len(objectsMetadata), Equals, 0)
+	c.Assert(resources.CommonPrefixes, DeepEquals, []string{})
+	c.Assert(resources.IsTruncated, Equals, false)
+}
+
+// test bucket list
+func (s *MyCacheSuite) TestMakeBucketAndList(c *C) {
+	// create bucket
+	err := dc.MakeBucket("foo2", "private")
+	c.Assert(err, IsNil)
+
+	// check bucket exists
+	buckets, err := dc.ListBuckets()
+	c.Assert(err, IsNil)
+	c.Assert(len(buckets), Equals, 5)
+	c.Assert(buckets[0].ACL, Equals, BucketACL("private"))
+}
+
+// test re-create bucket
+func (s *MyCacheSuite) TestMakeBucketWithSameNameFails(c *C) {
+	err := dc.MakeBucket("foo3", "private")
+	c.Assert(err, IsNil)
+
+	err = dc.MakeBucket("foo3", "private")
+	c.Assert(err, Not(IsNil))
+}
+
+// test make multiple buckets
+func (s *MyCacheSuite) TestCreateMultipleBucketsAndList(c *C) {
+	// add a second bucket
+	err := dc.MakeBucket("foo4", "private")
+	c.Assert(err, IsNil)
+
+	err = dc.MakeBucket("bar1", "private")
+	c.Assert(err, IsNil)
+
+	buckets, err := dc.ListBuckets()
+	c.Assert(err, IsNil)
+
+	c.Assert(len(buckets), Equals, 2)
+	c.Assert(buckets[0].Name, Equals, "bar1")
+	c.Assert(buckets[1].Name, Equals, "foo4")
+
+	err = dc.MakeBucket("foobar1", "private")
+	c.Assert(err, IsNil)
+
+	buckets, err = dc.ListBuckets()
+	c.Assert(err, IsNil)
+
+	c.Assert(len(buckets), Equals, 3)
+	c.Assert(buckets[2].Name, Equals, "foobar1")
+}
+
+// test object create without bucket
+func (s *MyCacheSuite) TestNewObjectFailsWithoutBucket(c *C) {
+	_, err := dc.CreateObject("unknown", "obj", "", "", 0, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test create object metadata
+func (s *MyCacheSuite) TestNewObjectMetadata(c *C) {
+	data := "Hello World"
+	hasher := md5.New()
+	hasher.Write([]byte(data))
+	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
+
+	err := dc.MakeBucket("foo6", "private")
+	c.Assert(err, IsNil)
+
+	objectMetadata, err := dc.CreateObject("foo6", "obj", "application/json", expectedMd5Sum, int64(len(data)), reader)
+	c.Assert(err, IsNil)
+	c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
+	c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json")
+}
+
+// test create object fails without name
+func (s *MyCacheSuite) TestNewObjectFailsWithEmptyName(c *C) {
+	_, err := dc.CreateObject("foo", "", "", "", 0, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test create object
+func (s *MyCacheSuite) TestNewObjectCanBeWritten(c *C) {
+	err := dc.MakeBucket("foo", "private")
+	c.Assert(err, IsNil)
+
+	data := "Hello World"
+
+	hasher := md5.New()
+	hasher.Write([]byte(data))
+	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
+
+	actualMetadata, err := dc.CreateObject("foo", "obj", "application/octet-stream", expectedMd5Sum, int64(len(data)), reader)
+	c.Assert(err, IsNil)
+	c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
+
+	var buffer bytes.Buffer
+	size, err := dc.GetObject(&buffer, "foo", "obj")
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len(data)))
+	c.Assert(buffer.Bytes(), DeepEquals, []byte(data))
+
+	actualMetadata, err = dc.GetObjectMetadata("foo", "obj")
+	c.Assert(err, IsNil)
+	c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum)
+	c.Assert(int64(len(data)), Equals, actualMetadata.Size)
+}
+
+// test list objects
+func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
+	c.Assert(dc.MakeBucket("foo5", "private"), IsNil)
+
+	one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
+
+	_, err := dc.CreateObject("foo5", "obj1", "", "", int64(len("one")), one)
+	c.Assert(err, IsNil)
+
+	two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
+	_, err = dc.CreateObject("foo5", "obj2", "", "", int64(len("two")), two)
+	c.Assert(err, IsNil)
+
+	var buffer1 bytes.Buffer
+	size, err := dc.GetObject(&buffer1, "foo5", "obj1")
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("one"))))
+	c.Assert(buffer1.Bytes(), DeepEquals, []byte("one"))
+
+	var buffer2 bytes.Buffer
+	size, err = dc.GetObject(&buffer2, "foo5", "obj2")
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("two"))))
+
+	c.Assert(buffer2.Bytes(), DeepEquals, []byte("two"))
+
+	/// test list of objects
+
+	// test list objects with prefix and delimiter
+	var resources BucketResourcesMetadata
+	resources.Prefix = "o"
+	resources.Delimiter = "1"
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err := dc.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
+
+	// test list objects with only delimiter
+	resources.Prefix = ""
+	resources.Delimiter = "1"
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(objectsMetadata[0].Object, Equals, "obj2")
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
+
+	// test list objects with only prefix
+	resources.Prefix = "o"
+	resources.Delimiter = ""
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(objectsMetadata[0].Object, Equals, "obj1")
+	c.Assert(objectsMetadata[1].Object, Equals, "obj2")
+
+	three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
+	_, err = dc.CreateObject("foo5", "obj3", "", "", int64(len("three")), three)
+	c.Assert(err, IsNil)
+
+	var buffer bytes.Buffer
+	size, err = dc.GetObject(&buffer, "foo5", "obj3")
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("three"))))
+	c.Assert(buffer.Bytes(), DeepEquals, []byte("three"))
+
+	// test list objects with maxkeys
+	resources.Prefix = "o"
+	resources.Delimiter = ""
+	resources.Maxkeys = 2
+	objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, true)
+	c.Assert(len(objectsMetadata), Equals, 2)
+}

pkg/storage/donut/donut_test.go

@@ -9,7 +9,7 @@
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliedd.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
@@ -31,13 +31,13 @@ import (
 	. "github.com/minio/check"
 )
 
-func Test(t *testing.T) { TestingT(t) }
+func TestDonut(t *testing.T) { TestingT(t) }
 
-type MySuite struct {
+type MyDonutSuite struct {
 	root string
 }
 
-var _ = Suite(&MySuite{})
+var _ = Suite(&MyDonutSuite{})
 
 // create a dummy TestNodeDiskMap
 func createTestNodeDiskMap(p string) map[string][]string {
@@ -55,39 +55,40 @@ func createTestNodeDiskMap(p string) map[string][]string {
 	return nodes
 }
 
-var d Cache
+var dd Cache
 
-func (s *MySuite) SetUpSuite(c *C) {
+func (s *MyDonutSuite) SetUpSuite(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	s.root = root
-	d = NewCache(100000, time.Duration(1*time.Hour), "test", createTestNodeDiskMap(root))
+	dd = NewCache(100000, time.Duration(1*time.Hour), "test", createTestNodeDiskMap(root))
-	buckets, err := d.ListBuckets()
+	// testing empty donut
+	buckets, err := dd.ListBuckets()
 	c.Assert(err, IsNil)
 	c.Assert(len(buckets), Equals, 0)
 }
 
-func (s *MySuite) TearDownSuite(c *C) {
+func (s *MyDonutSuite) TearDownSuite(c *C) {
 	os.RemoveAll(s.root)
 }
 
 // test make bucket without name
-func (s *MySuite) TestBucketWithoutNameFails(c *C) {
+func (s *MyDonutSuite) TestBucketWithoutNameFails(c *C) {
 	// fail to create new bucket without a name
-	err := d.MakeBucket("", "private")
+	err := dd.MakeBucket("", "private")
 	c.Assert(err, Not(IsNil))
 
-	err = d.MakeBucket(" ", "private")
+	err = dd.MakeBucket(" ", "private")
 	c.Assert(err, Not(IsNil))
 }
 
 // test empty bucket
-func (s *MySuite) TestEmptyBucket(c *C) {
+func (s *MyDonutSuite) TestEmptyBucket(c *C) {
-	c.Assert(d.MakeBucket("foo1", "private"), IsNil)
+	c.Assert(dd.MakeBucket("foo1", "private"), IsNil)
 	// check if bucket is empty
 	var resources BucketResourcesMetadata
 	resources.Maxkeys = 1
-	objectsMetadata, resources, err := d.ListObjects("foo1", resources)
+	objectsMetadata, resources, err := dd.ListObjects("foo1", resources)
 	c.Assert(err, IsNil)
 	c.Assert(len(objectsMetadata), Equals, 0)
 	c.Assert(resources.CommonPrefixes, DeepEquals, []string{})
@@ -95,47 +96,47 @@ func (s *MySuite) TestEmptyBucket(c *C) {
 }
 
 // test bucket list
-func (s *MySuite) TestMakeBucketAndList(c *C) {
+func (s *MyDonutSuite) TestMakeBucketAndList(c *C) {
 	// create bucket
-	err := d.MakeBucket("foo2", "private")
+	err := dd.MakeBucket("foo2", "private")
 	c.Assert(err, IsNil)
 
 	// check bucket exists
-	buckets, err := d.ListBuckets()
+	buckets, err := dd.ListBuckets()
 	c.Assert(err, IsNil)
 	c.Assert(len(buckets), Equals, 5)
 	c.Assert(buckets[0].ACL, Equals, BucketACL("private"))
 }
 
 // test re-create bucket
-func (s *MySuite) TestMakeBucketWithSameNameFails(c *C) {
+func (s *MyDonutSuite) TestMakeBucketWithSameNameFails(c *C) {
-	err := d.MakeBucket("foo3", "private")
+	err := dd.MakeBucket("foo3", "private")
 	c.Assert(err, IsNil)
 
-	err = d.MakeBucket("foo3", "private")
+	err = dd.MakeBucket("foo3", "private")
 	c.Assert(err, Not(IsNil))
 }
 
 // test make multiple buckets
-func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
+func (s *MyDonutSuite) TestCreateMultipleBucketsAndList(c *C) {
 	// add a second bucket
-	err := d.MakeBucket("foo4", "private")
+	err := dd.MakeBucket("foo4", "private")
 	c.Assert(err, IsNil)
 
-	err = d.MakeBucket("bar1", "private")
+	err = dd.MakeBucket("bar1", "private")
 	c.Assert(err, IsNil)
 
-	buckets, err := d.ListBuckets()
+	buckets, err := dd.ListBuckets()
 	c.Assert(err, IsNil)
 
 	c.Assert(len(buckets), Equals, 2)
 	c.Assert(buckets[0].Name, Equals, "bar1")
 	c.Assert(buckets[1].Name, Equals, "foo4")
 
-	err = d.MakeBucket("foobar1", "private")
+	err = dd.MakeBucket("foobar1", "private")
 	c.Assert(err, IsNil)
 
-	buckets, err = d.ListBuckets()
+	buckets, err = dd.ListBuckets()
 	c.Assert(err, IsNil)
 
 	c.Assert(len(buckets), Equals, 3)
@@ -143,37 +144,37 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
 }
 
 // test object create without bucket
-func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
+func (s *MyDonutSuite) TestNewObjectFailsWithoutBucket(c *C) {
-	_, err := d.CreateObject("unknown", "obj", "", "", 0, nil)
+	_, err := dd.CreateObject("unknown", "obj", "", "", 0, nil)
 	c.Assert(err, Not(IsNil))
 }
 
 // test create object metadata
-func (s *MySuite) TestNewObjectMetadata(c *C) {
+func (s *MyDonutSuite) TestNewObjectMetadata(c *C) {
 	data := "Hello World"
 	hasher := md5.New()
 	hasher.Write([]byte(data))
 	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
 	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
 
-	err := d.MakeBucket("foo6", "private")
+	err := dd.MakeBucket("foo6", "private")
 	c.Assert(err, IsNil)
 
-	objectMetadata, err := d.CreateObject("foo6", "obj", "application/json", expectedMd5Sum, int64(len(data)), reader)
+	objectMetadata, err := dd.CreateObject("foo6", "obj", "application/json", expectedMd5Sum, int64(len(data)), reader)
 	c.Assert(err, IsNil)
 	c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
 	c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json")
 }
 
 // test create object fails without name
-func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
+func (s *MyDonutSuite) TestNewObjectFailsWithEmptyName(c *C) {
-	_, err := d.CreateObject("foo", "", "", "", 0, nil)
+	_, err := dd.CreateObject("foo", "", "", "", 0, nil)
 	c.Assert(err, Not(IsNil))
 }
 
 // test create object
-func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
+func (s *MyDonutSuite) TestNewObjectCanBeWritten(c *C) {
-	err := d.MakeBucket("foo", "private")
+	err := dd.MakeBucket("foo", "private")
 	c.Assert(err, IsNil)
 
 	data := "Hello World"
@@ -183,43 +184,43 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
 	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
 	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
 
-	actualMetadata, err := d.CreateObject("foo", "obj", "application/octet-stream", expectedMd5Sum, int64(len(data)), reader)
+	actualMetadata, err := dd.CreateObject("foo", "obj", "application/octet-stream", expectedMd5Sum, int64(len(data)), reader)
 	c.Assert(err, IsNil)
 	c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
 
 	var buffer bytes.Buffer
-	size, err := d.GetObject(&buffer, "foo", "obj")
+	size, err := dd.GetObject(&buffer, "foo", "obj")
 	c.Assert(err, IsNil)
 	c.Assert(size, Equals, int64(len(data)))
 	c.Assert(buffer.Bytes(), DeepEquals, []byte(data))
 
-	actualMetadata, err = d.GetObjectMetadata("foo", "obj")
+	actualMetadata, err = dd.GetObjectMetadata("foo", "obj")
 	c.Assert(err, IsNil)
 	c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum)
 	c.Assert(int64(len(data)), Equals, actualMetadata.Size)
 }
 
 // test list objects
-func (s *MySuite) TestMultipleNewObjects(c *C) {
+func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {
-	c.Assert(d.MakeBucket("foo5", "private"), IsNil)
+	c.Assert(dd.MakeBucket("foo5", "private"), IsNil)
 
 	one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
 
-	_, err := d.CreateObject("foo5", "obj1", "", "", int64(len("one")), one)
+	_, err := dd.CreateObject("foo5", "obj1", "", "", int64(len("one")), one)
 	c.Assert(err, IsNil)
 
 	two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
-	_, err = d.CreateObject("foo5", "obj2", "", "", int64(len("two")), two)
+	_, err = dd.CreateObject("foo5", "obj2", "", "", int64(len("two")), two)
 	c.Assert(err, IsNil)
 
 	var buffer1 bytes.Buffer
-	size, err := d.GetObject(&buffer1, "foo5", "obj1")
+	size, err := dd.GetObject(&buffer1, "foo5", "obj1")
 	c.Assert(err, IsNil)
 	c.Assert(size, Equals, int64(len([]byte("one"))))
 	c.Assert(buffer1.Bytes(), DeepEquals, []byte("one"))
 
 	var buffer2 bytes.Buffer
-	size, err = d.GetObject(&buffer2, "foo5", "obj2")
+	size, err = dd.GetObject(&buffer2, "foo5", "obj2")
 	c.Assert(err, IsNil)
 	c.Assert(size, Equals, int64(len([]byte("two"))))
 
@@ -232,7 +233,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
 	resources.Prefix = "o"
 	resources.Delimiter = "1"
 	resources.Maxkeys = 10
-	objectsMetadata, resources, err := d.ListObjects("foo5", resources)
+	objectsMetadata, resources, err := dd.ListObjects("foo5", resources)
 	c.Assert(err, IsNil)
 	c.Assert(resources.IsTruncated, Equals, false)
 	c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
@@ -241,9 +242,9 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
 	resources.Prefix = ""
 	resources.Delimiter = "1"
 	resources.Maxkeys = 10
-	objectsMetadata, resources, err = d.ListObjects("foo5", resources)
+	objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
 	c.Assert(err, IsNil)
-	c.Assert(objectsMetadata[0].Object, Equals, "obj1")
+	c.Assert(objectsMetadata[0].Object, Equals, "obj2")
 	c.Assert(resources.IsTruncated, Equals, false)
 	c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
 
@@ -251,18 +252,18 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
 	resources.Prefix = "o"
 	resources.Delimiter = ""
 	resources.Maxkeys = 10
-	objectsMetadata, resources, err = d.ListObjects("foo5", resources)
+	objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
 	c.Assert(err, IsNil)
 	c.Assert(resources.IsTruncated, Equals, false)
 	c.Assert(objectsMetadata[0].Object, Equals, "obj1")
 	c.Assert(objectsMetadata[1].Object, Equals, "obj2")
 
 	three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
-	_, err = d.CreateObject("foo5", "obj3", "", "", int64(len("three")), three)
+	_, err = dd.CreateObject("foo5", "obj3", "", "", int64(len("three")), three)
 	c.Assert(err, IsNil)
 
 	var buffer bytes.Buffer
-	size, err = d.GetObject(&buffer, "foo5", "obj3")
+	size, err = dd.GetObject(&buffer, "foo5", "obj3")
 	c.Assert(err, IsNil)
 	c.Assert(size, Equals, int64(len([]byte("three"))))
 	c.Assert(buffer.Bytes(), DeepEquals, []byte("three"))
@@ -271,7 +272,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
 	resources.Prefix = "o"
 	resources.Delimiter = ""
 	resources.Maxkeys = 2
-	objectsMetadata, resources, err = d.ListObjects("foo5", resources)
+	objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
 	c.Assert(err, IsNil)
 	c.Assert(resources.IsTruncated, Equals, true)
 	c.Assert(len(objectsMetadata), Equals, 2)

pkg/storage/pq/pq.go

@@ -1,70 +0,0 @@
-/*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package pq
-
-import "container/heap"
-
-// Item container for tasks in priority queue
-type Item struct {
-	task Task // task
-
-	// The index is needed by Fix and is maintained by the heap.Interface methods.
-	index int // The index of the item in the heap.
-}
-
-// A PriorityQueue implements heap.Interface and holds Items.
-type PriorityQueue []*Item
-
-// Len length of current priority queue
-func (pq PriorityQueue) Len() int { return len(pq) }
-
-// Less used internally by heap.Interface to arrange items in order
-func (pq PriorityQueue) Less(i, j int) bool {
-	// We want Pop to give us the highest, not lowest, priority so we use greater than here.
-	return pq[i].task.GetPriority() > pq[j].task.GetPriority()
-}
-
-// Swap used internally by heap.Interface to arrange incoming items
-func (pq PriorityQueue) Swap(i, j int) {
-	pq[i], pq[j] = pq[j], pq[i]
-	pq[i].index = i
-	pq[j].index = j
-}
-
-// Push push items onto priority queue
-func (pq *PriorityQueue) Push(x interface{}) {
-	n := len(*pq)
-	item := x.(*Item)
-	item.index = n
-	*pq = append(*pq, item)
-}
-
-// Pop pop items with highest priority
-func (pq *PriorityQueue) Pop() interface{} {
-	old := *pq
-	n := len(old)
-	item := old[n-1]
-	item.index = -1 // for safety
-	*pq = old[0 : n-1]
-	return item
-}
-
-// Fix modifies an item in-place on the queue
-func (pq *PriorityQueue) Fix(item *Item, task Task) {
-	item.task = task
-	heap.Fix(pq, item.index)
-}

pkg/storage/pq/pq_test.go

@@ -1,101 +0,0 @@
-/*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package pq
-
-import (
-	"container/heap"
-	"fmt"
-	"testing"
-
-	. "github.com/minio/check"
-)
-
-func Test(t *testing.T) { TestingT(t) }
-
-type MySuite struct{}
-
-var _ = Suite(&MySuite{})
-
-func helloTask1() <-chan error {
-	errCh := make(chan error)
-	go func() {
-		defer close(errCh)
-		println("Hello task1")
-		errCh <- nil
-	}()
-	return errCh
-}
-
-func helloTask2() <-chan error {
-	errCh := make(chan error)
-	go func() {
-		defer close(errCh)
-		println("Hello task2")
-		errCh <- nil
-	}()
-	return errCh
-}
-
-func newJob1() <-chan error {
-	errCh := make(chan error)
-	go func() {
-		defer close(errCh)
-		println("New Job1")
-		errCh <- nil
-	}()
-	return errCh
-}
-
-func newJob2() <-chan error {
-	errCh := make(chan error)
-	go func() {
-		defer close(errCh)
-		println("New Job2")
-		errCh <- nil
-	}()
-	return errCh
-}
-
-func (s *MySuite) TestPQ(c *C) {
-	// Create a priority queue, put the items in it, and
-	// establish the priority queue (heap) invariants.
-	pq := make(PriorityQueue, 2)
-	pq[0] = &Item{
-		task:  Task{job: helloTask1, priority: 2},
-		index: 0,
-	}
-	pq[1] = &Item{
-		task:  Task{job: helloTask2, priority: 1},
-		index: 1,
-	}
-	heap.Init(&pq)
-
-	// Insert a new item and then modify its priority.
-	item := &Item{
-		task: Task{job: newJob1, priority: 5},
-	}
-	heap.Push(&pq, item)
-	newTask := Task{job: newJob2, priority: 6}
-	pq.Fix(item, newTask)
-
-	// Take the items out; they arrive in decreasing priority order.
-	for pq.Len() > 0 {
-		item := heap.Pop(&pq).(*Item)
-		fmt.Printf("%.2d", item.task.GetPriority())
-		item.task.Execute()
-	}
-}

pkg/storage/pq/task.go

@@ -1,38 +0,0 @@
-/*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package pq
-
-// Task container for any generic tasks
-type Task struct {
-	job      func() <-chan error
-	priority int
-}
-
-// GetPriority get current task priority
-func (t Task) GetPriority() int {
-	return t.priority
-}
-
-// UpdatePriority update current task priority
-func (t Task) UpdatePriority(p int) {
-	t.priority = p
-}
-
-// Execute execute current task
-func (t Task) Execute() error {
-	return <-t.job()
-}