Finishing all the test case support for ACL and other fixes

pkg/storage/drivers/api_testsuite.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/minio-io/check"
+	"github.com/minio-io/minio/pkg/iodine"
 )
 
 // APITestSuite - collection of API tests
@@ -105,6 +106,7 @@ func testPaging(c *check.C, create func() Driver) {
 		key := "obj" + strconv.Itoa(i)
 		drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
 		resources.Maxkeys = 5
+		resources.Prefix = ""
 		objects, resources, err = drivers.ListObjects("bucket", resources)
 		c.Assert(len(objects), check.Equals, i+1)
 		c.Assert(resources.IsTruncated, check.Equals, false)
@@ -115,6 +117,7 @@ func testPaging(c *check.C, create func() Driver) {
 		key := "obj" + strconv.Itoa(i)
 		drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
 		resources.Maxkeys = 5
+		resources.Prefix = ""
 		objects, resources, err = drivers.ListObjects("bucket", resources)
 		c.Assert(len(objects), check.Equals, 5)
 		c.Assert(resources.IsTruncated, check.Equals, true)
@@ -329,7 +332,7 @@ func testNonExistantObjectInBucket(c *check.C, create func() Driver) {
 	c.Assert(length, check.Equals, int64(0))
 	c.Assert(err, check.Not(check.IsNil))
 	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
-	switch err := err.(type) {
+	switch err := iodine.ToError(err).(type) {
 	case ObjectNotFound:
 		{
 			c.Assert(err, check.ErrorMatches, "Object not Found: bucket#dir1")
@@ -352,7 +355,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
 	var byteBuffer bytes.Buffer
 	length, err := drivers.GetObject(&byteBuffer, "bucket", "dir1")
 	c.Assert(length, check.Equals, int64(0))
-	switch err := err.(type) {
+	switch err := iodine.ToError(err).(type) {
 	case ObjectNotFound:
 		{
 			c.Assert(err.Bucket, check.Equals, "bucket")
@@ -369,7 +372,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
 	var byteBuffer2 bytes.Buffer
 	length, err = drivers.GetObject(&byteBuffer, "bucket", "dir1/")
 	c.Assert(length, check.Equals, int64(0))
-	switch err := err.(type) {
+	switch err := iodine.ToError(err).(type) {
 	case ObjectNotFound:
 		{
 			c.Assert(err.Bucket, check.Equals, "bucket")

pkg/storage/drivers/donut/donut.go

@@ -19,7 +19,6 @@ package donut
 import (
 	"encoding/base64"
 	"encoding/hex"
-	"errors"
 	"io"
 	"os"
 	"path"
@@ -136,7 +135,7 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat
 	}
 	metadata, err := d.donut.GetBucketMetadata(bucketName)
 	if err != nil {
-		return drivers.BucketMetadata{}, drivers.BucketNotFound{Bucket: bucketName}
+		return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucketName}, nil)
 	}
 	created, err := time.Parse(time.RFC3339Nano, metadata["created"])
 	if err != nil {
@@ -156,25 +155,21 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat
 
 // GetObject retrieves an object and writes it to a writer
 func (d donutDriver) GetObject(target io.Writer, bucketName, objectName string) (int64, error) {
-	errParams := map[string]string{
-		"bucketName": bucketName,
-		"objectName": objectName,
+	if !drivers.IsValidBucket(bucketName) || strings.Contains(bucketName, ".") {
+		return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
 	}
-	if bucketName == "" || strings.TrimSpace(bucketName) == "" {
-		return 0, iodine.New(errors.New("invalid argument"), errParams)
-	}
-	if objectName == "" || strings.TrimSpace(objectName) == "" {
-		return 0, iodine.New(errors.New("invalid argument"), errParams)
+	if !drivers.IsValidObject(objectName) || strings.TrimSpace(objectName) == "" {
+		return 0, iodine.New(drivers.ObjectNameInvalid{Object: objectName}, nil)
 	}
 	reader, size, err := d.donut.GetObject(bucketName, objectName)
 	if err != nil {
-		return 0, drivers.ObjectNotFound{
+		return 0, iodine.New(drivers.ObjectNotFound{
 			Bucket: bucketName,
 			Object: objectName,
-		}
+		}, nil)
 	}
 	n, err := io.CopyN(target, reader, size)
-	return n, iodine.New(err, errParams)
+	return n, iodine.New(err, nil)
 }
 
 // GetPartialObject retrieves an object range and writes it to a writer
@@ -186,25 +181,31 @@ func (d donutDriver) GetPartialObject(w io.Writer, bucketName, objectName string
 		"start":      strconv.FormatInt(start, 10),
 		"length":     strconv.FormatInt(length, 10),
 	}
-	if bucketName == "" || strings.TrimSpace(bucketName) == "" {
-		return 0, iodine.New(errors.New("invalid argument"), errParams)
+	if !drivers.IsValidBucket(bucketName) || strings.Contains(bucketName, ".") {
+		return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, errParams)
 	}
-	if objectName == "" || strings.TrimSpace(objectName) == "" {
-		return 0, iodine.New(errors.New("invalid argument"), errParams)
+	if !drivers.IsValidObject(objectName) || strings.TrimSpace(objectName) == "" {
+		return 0, iodine.New(drivers.ObjectNameInvalid{Object: objectName}, errParams)
 	}
 	if start < 0 {
-		return 0, iodine.New(errors.New("invalid argument"), errParams)
+		return 0, iodine.New(drivers.InvalidRange{
+			Start:  start,
+			Length: length,
+		}, errParams)
 	}
 	reader, size, err := d.donut.GetObject(bucketName, objectName)
 	defer reader.Close()
 	if err != nil {
-		return 0, drivers.ObjectNotFound{
+		return 0, iodine.New(drivers.ObjectNotFound{
 			Bucket: bucketName,
 			Object: objectName,
-		}
+		}, nil)
 	}
 	if start > size || (start+length-1) > size {
-		return 0, iodine.New(errors.New("invalid range"), errParams)
+		return 0, iodine.New(drivers.InvalidRange{
+			Start:  start,
+			Length: length,
+		}, errParams)
 	}
 	_, err = io.CopyN(ioutil.Discard, reader, start)
 	if err != nil {
@@ -224,6 +225,12 @@ func (d donutDriver) GetObjectMetadata(bucketName, objectName, prefixName string
 		"objectName": objectName,
 		"prefixName": prefixName,
 	}
+	if !drivers.IsValidBucket(bucketName) || strings.Contains(bucketName, ".") {
+		return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
+	}
+	if !drivers.IsValidObject(objectName) || strings.TrimSpace(objectName) == "" {
+		return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNameInvalid{Object: objectName}, nil)
+	}
 	metadata, err := d.donut.GetObjectMetadata(bucketName, objectName)
 	if err != nil {
 		return drivers.ObjectMetadata{}, drivers.ObjectNotFound{
@@ -262,6 +269,12 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso
 	errParams := map[string]string{
 		"bucketName": bucketName,
 	}
+	if !drivers.IsValidBucket(bucketName) || strings.Contains(bucketName, ".") {
+		return nil, drivers.BucketResourcesMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
+	}
+	if !drivers.IsValidObject(resources.Prefix) {
+		return nil, drivers.BucketResourcesMetadata{}, iodine.New(drivers.ObjectNameInvalid{Object: resources.Prefix}, nil)
+	}
 	actualObjects, commonPrefixes, isTruncated, err := d.donut.ListObjects(bucketName,
 		resources.Prefix,
 		resources.Marker,
@@ -305,11 +318,11 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
 		"objectName":  objectName,
 		"contentType": contentType,
 	}
-	if bucketName == "" || strings.TrimSpace(bucketName) == "" {
-		return iodine.New(errors.New("invalid argument"), errParams)
+	if !drivers.IsValidBucket(bucketName) || strings.Contains(bucketName, ".") {
+		return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
 	}
-	if objectName == "" || strings.TrimSpace(objectName) == "" {
-		return iodine.New(errors.New("invalid argument"), errParams)
+	if !drivers.IsValidObject(objectName) || strings.TrimSpace(objectName) == "" {
+		return iodine.New(drivers.ObjectNameInvalid{Object: objectName}, nil)
 	}
 	if strings.TrimSpace(contentType) == "" {
 		contentType = "application/octet-stream"
@@ -324,7 +337,6 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
 		}
 		expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
 	}
-
 	err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, ioutil.NopCloser(reader), metadata)
 	if err != nil {
 		return iodine.New(err, errParams)

pkg/storage/drivers/driver.go

@@ -19,6 +19,7 @@ package drivers
 import (
 	"io"
 	"regexp"
+	"strings"
 	"time"
 	"unicode/utf8"
 )
@@ -185,6 +186,9 @@ func IsValidBucket(bucket string) bool {
 // IsValidObject - verify object name in accordance with
 //   - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
 func IsValidObject(object string) bool {
+	if strings.TrimSpace(object) == "" {
+		return true
+	}
 	if len(object) > 1024 || len(object) == 0 {
 		return false
 	}

pkg/storage/drivers/errors.go

@@ -16,6 +16,8 @@
 
 package drivers
 
+import "fmt"
+
 // BackendError - generic disk backend error
 type BackendError struct {
 	Path string
@@ -164,3 +166,23 @@ func (e BadDigest) Error() string {
 func (e InvalidDigest) Error() string {
 	return "Md5 provided " + e.Md5 + " is invalid"
 }
+
+// OperationNotPermitted - operation not permitted
+type OperationNotPermitted struct {
+	Op     string
+	Reason string
+}
+
+func (e OperationNotPermitted) Error() string {
+	return "Operation " + e.Op + " not permitted for reason: " + e.Reason
+}
+
+// InvalidRange - invalid range
+type InvalidRange struct {
+	Start  int64
+	Length int64
+}
+
+func (e InvalidRange) Error() string {
+	return fmt.Sprintf("Invalid range start:%d length:%d", e.Start, e.Length)
+}

pkg/storage/drivers/memory/memory.go

@@ -25,14 +25,14 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio-io/minio/pkg/storage/drivers"
-
 	"crypto/md5"
 	"encoding/hex"
 
 	"io/ioutil"
 
 	"github.com/golang/groupcache/lru"
+	"github.com/minio-io/minio/pkg/iodine"
+	"github.com/minio-io/minio/pkg/storage/drivers"
 	"github.com/minio-io/minio/pkg/utils/log"
 )
 
@@ -91,8 +91,14 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error) {
 func (memory *memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
+	if !drivers.IsValidBucket(bucket) {
+		return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
+	}
+	if !drivers.IsValidObject(object) {
+		return 0, iodine.New(drivers.ObjectNameInvalid{Object: object}, nil)
+	}
 	if _, ok := memory.bucketMetadata[bucket]; ok == false {
-		return 0, drivers.BucketNotFound{Bucket: bucket}
+		return 0, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
 	}
 	// get object
 	objectKey := bucket + "/" + object
@@ -101,10 +107,10 @@ func (memory *memoryDriver) GetObject(w io.Writer, bucket string, object string)
 			dataSlice := data.([]byte)
 			objectBuffer := bytes.NewBuffer(dataSlice)
 			written, err := io.Copy(w, objectBuffer)
-			return written, err
+			return written, iodine.New(err, nil)
 		}
 	}
-	return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
+	return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil)
 }
 
 // GetPartialObject - GET object from memory buffer range
@@ -112,11 +118,17 @@ func (memory *memoryDriver) GetPartialObject(w io.Writer, bucket, object string,
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
 	var sourceBuffer bytes.Buffer
+	if !drivers.IsValidBucket(bucket) {
+		return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
+	}
+	if !drivers.IsValidObject(object) {
+		return 0, iodine.New(drivers.ObjectNameInvalid{Object: object}, nil)
+	}
 	if _, err := memory.GetObject(&sourceBuffer, bucket, object); err != nil {
-		return 0, err
+		return 0, iodine.New(err, nil)
 	}
 	if _, err := io.CopyN(ioutil.Discard, &sourceBuffer, start); err != nil {
-		return 0, err
+		return 0, iodine.New(err, nil)
 	}
 	return io.CopyN(w, &sourceBuffer, length)
 }
@@ -125,8 +137,11 @@ func (memory *memoryDriver) GetPartialObject(w io.Writer, bucket, object string,
 func (memory *memoryDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
+	if !drivers.IsValidBucket(bucket) {
+		return drivers.BucketMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
+	}
 	if _, ok := memory.bucketMetadata[bucket]; ok == false {
-		return drivers.BucketMetadata{}, drivers.BucketNotFound{Bucket: bucket}
+		return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
 	}
 	return memory.bucketMetadata[bucket].metadata, nil
 }
@@ -134,17 +149,22 @@ func (memory *memoryDriver) GetBucketMetadata(bucket string) (drivers.BucketMeta
 // CreateObject - PUT object to memory buffer
 func (memory *memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
 	memory.lock.RLock()
-
+	if !drivers.IsValidBucket(bucket) {
+		memory.lock.RUnlock()
+		return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
+	}
+	if !drivers.IsValidObject(key) {
+		memory.lock.RUnlock()
+		return iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
+	}
 	if _, ok := memory.bucketMetadata[bucket]; ok == false {
 		memory.lock.RUnlock()
-		return drivers.BucketNotFound{Bucket: bucket}
+		return iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
 	}
-
 	objectKey := bucket + "/" + key
-
 	if _, ok := memory.objectMetadata[objectKey]; ok == true {
 		memory.lock.RUnlock()
-		return drivers.ObjectExists{Bucket: bucket, Object: key}
+		return iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil)
 	}
 	memory.lock.RUnlock()
 
@@ -175,7 +195,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, md5sum string
 	memory.lock.Lock()
 	if _, ok := memory.objectMetadata[objectKey]; ok == true {
 		memory.lock.Unlock()
-		return drivers.ObjectExists{Bucket: bucket, Object: key}
+		return iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil)
 	}
 	memory.objectMetadata[objectKey] = newObject
 	memory.objects.Add(objectKey, dataSlice)
@@ -192,15 +212,15 @@ func (memory *memoryDriver) CreateBucket(bucketName, acl string) error {
 	memory.lock.RLock()
 	if !drivers.IsValidBucket(bucketName) {
 		memory.lock.RUnlock()
-		return drivers.BucketNameInvalid{Bucket: bucketName}
+		return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
 	}
 	if !drivers.IsValidBucketACL(acl) {
 		memory.lock.RUnlock()
-		return drivers.InvalidACL{ACL: acl}
+		return iodine.New(drivers.InvalidACL{ACL: acl}, nil)
 	}
 	if _, ok := memory.bucketMetadata[bucketName]; ok == true {
 		memory.lock.RUnlock()
-		return drivers.BucketExists{Bucket: bucketName}
+		return iodine.New(drivers.BucketExists{Bucket: bucketName}, nil)
 	}
 	memory.lock.RUnlock()
 
@@ -247,42 +267,53 @@ func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedN
 	return resources, keys
 }
 
+func (memory *memoryDriver) listObjectsInternal(keys []string, key string, resources drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) {
+	switch true {
+	// Prefix absent, delimit object key based on delimiter
+	case resources.IsDelimiterSet():
+		delimitedName := delimiter(key, resources.Delimiter)
+		switch true {
+		case delimitedName == "" || delimitedName == key:
+			keys = appendUniq(keys, key)
+		case delimitedName != "":
+			resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
+		}
+	// Prefix present, delimit object key with prefix key based on delimiter
+	case resources.IsDelimiterPrefixSet():
+		if strings.HasPrefix(key, resources.Prefix) {
+			trimmedName := strings.TrimPrefix(key, resources.Prefix)
+			delimitedName := delimiter(trimmedName, resources.Delimiter)
+			resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
+		}
+	// Prefix present, nothing to delimit
+	case resources.IsPrefixSet():
+		keys = appendUniq(keys, key)
+	// Prefix and delimiter absent
+	case resources.IsDefault():
+		keys = appendUniq(keys, key)
+	}
+	return keys, resources
+}
+
 // ListObjects - list objects from memory
 func (memory *memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
+	if !drivers.IsValidBucket(bucket) {
+		return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
+	}
+	if !drivers.IsValidObject(resources.Prefix) {
+		return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.ObjectNameInvalid{Object: resources.Prefix}, nil)
+	}
 	if _, ok := memory.bucketMetadata[bucket]; ok == false {
-		return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
+		return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
 	}
 	var results []drivers.ObjectMetadata
 	var keys []string
 	for key := range memory.objectMetadata {
 		if strings.HasPrefix(key, bucket+"/") {
 			key = key[len(bucket)+1:]
-			switch true {
-			// Prefix absent, delimit object key based on delimiter
-			case resources.IsDelimiterSet():
-				delimitedName := delimiter(key, resources.Delimiter)
-				switch true {
-				case delimitedName == "" || delimitedName == key:
-					keys = appendUniq(keys, key)
-				case delimitedName != "":
-					resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
-				}
-			// Prefix present, delimit object key with prefix key based on delimiter
-			case resources.IsDelimiterPrefixSet():
-				if strings.HasPrefix(key, resources.Prefix) {
-					trimmedName := strings.TrimPrefix(key, resources.Prefix)
-					delimitedName := delimiter(trimmedName, resources.Delimiter)
-					resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
-				}
-			// Prefix present, nothing to delimit
-			case resources.IsPrefixSet():
-				keys = appendUniq(keys, key)
-			// Prefix and delimiter absent
-			case resources.IsDefault():
-				keys = appendUniq(keys, key)
-			}
+			keys, resources = memory.listObjectsInternal(keys, key, resources)
 		}
 	}
 	sort.Strings(keys)
@@ -327,21 +358,24 @@ func (memory *memoryDriver) GetObjectMetadata(bucket, key, prefix string) (drive
 	memory.lock.RLock()
 	defer memory.lock.RUnlock()
 	// check if bucket exists
-	if _, ok := memory.bucketMetadata[bucket]; ok == false {
-		return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket}
+	if !drivers.IsValidBucket(bucket) {
+		return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
+	}
+	if !drivers.IsValidObject(key) || !drivers.IsValidObject(prefix) {
+		return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
+	}
+	if _, ok := memory.bucketMetadata[bucket]; ok == false {
+		return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
 	}
-
 	objectKey := bucket + "/" + key
-
 	if object, ok := memory.objectMetadata[objectKey]; ok == true {
 		return object.metadata, nil
 	}
-	return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
+	return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: key}, nil)
 }
 
 func (memory *memoryDriver) evictObject(key lru.Key, value interface{}) {
 	k := key.(string)
-
 	memory.totalSize = memory.totalSize - memory.objectMetadata[k].metadata.Size
 	log.Println("evicting:", k)
 	delete(memory.objectMetadata, k)