ACL driver/storage layer support

pkg/api/api_generic_handlers.go

@@ -79,6 +79,10 @@ func (h vHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			writeErrorResponse(w, r, AccessDenied, acceptsContentType, r.URL.Path)
 			return
 		}
+		if r.Method == "PUT" && bucketMetadata.ACL.IsPublicRead() {
+			writeErrorResponse(w, r, AccessDenied, acceptsContentType, r.URL.Path)
+			return
+		}
 	}
 
 	switch true {

pkg/storage/donut/donut.go

@@ -31,9 +31,13 @@ type donut struct {
 
 // config files used inside Donut
 const (
+	// donut object metadata and config
 	donutObjectMetadataConfig = "donutObjectMetadata.json"
-	objectMetadataConfig      = "objectMetadata.json"
 	donutConfig               = "donutMetadata.json"
+
+	// bucket, object metadata
+	bucketMetadataConfig = "bucketMetadata.json"
+	objectMetadataConfig = "objectMetadata.json"
 )
 
 // attachDonutNode - wrapper function to instantiate a new node for associated donut

pkg/storage/donut/donut_bucket.go

@@ -35,22 +35,27 @@ import (
 // internal struct carrying bucket specific information
 type bucket struct {
 	name      string
+	acl       string
+	time      time.Time
 	donutName string
 	nodes     map[string]Node
 	objects   map[string]Object
 }
 
 // NewBucket - instantiate a new bucket
-func NewBucket(bucketName, donutName string, nodes map[string]Node) (Bucket, error) {
+func NewBucket(bucketName, aclType, donutName string, nodes map[string]Node) (Bucket, error) {
 	errParams := map[string]string{
 		"bucketName": bucketName,
 		"donutName":  donutName,
+		"aclType":    aclType,
 	}
 	if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(donutName) == "" {
 		return nil, iodine.New(errors.New("invalid argument"), errParams)
 	}
 	b := bucket{}
 	b.name = bucketName
+	b.acl = aclType
+	b.time = time.Now()
 	b.donutName = donutName
 	b.objects = make(map[string]Object)
 	b.nodes = nodes

pkg/storage/donut/donut_public_interfaces.go

@@ -32,7 +32,7 @@ type ObjectStorage interface {
 	GetBucketMetadata(bucket string) (map[string]string, error)
 	SetBucketMetadata(bucket string, metadata map[string]string) error
 	ListBuckets() ([]string, error)
-	MakeBucket(bucket string) error
+	MakeBucket(bucket, acl string) error
 
 	// Bucket Operations
 	ListObjects(bucket, prefix, marker, delim string, maxKeys int) (result []string, prefixes []string, isTruncated bool, err error)

pkg/storage/donut/donut_test.go

@@ -75,10 +75,10 @@ func (s *MySuite) TestBucketWithoutNameFails(c *C) {
 	donut, err := NewDonut("test", createTestNodeDiskMap(root))
 	c.Assert(err, IsNil)
 	// fail to create new bucket without a name
-	err = donut.MakeBucket("")
+	err = donut.MakeBucket("", "private")
 	c.Assert(err, Not(IsNil))
 
-	err = donut.MakeBucket(" ")
+	err = donut.MakeBucket(" ", "private")
 	c.Assert(err, Not(IsNil))
 }
 
@@ -90,7 +90,7 @@ func (s *MySuite) TestEmptyBucket(c *C) {
 	donut, err := NewDonut("test", createTestNodeDiskMap(root))
 	c.Assert(err, IsNil)
 
-	c.Assert(donut.MakeBucket("foo"), IsNil)
+	c.Assert(donut.MakeBucket("foo", "private"), IsNil)
 	// check if bucket is empty
 	objects, _, istruncated, err := donut.ListObjects("foo", "", "", "", 1)
 	c.Assert(err, IsNil)
@@ -106,7 +106,7 @@ func (s *MySuite) TestMakeBucketAndList(c *C) {
 	donut, err := NewDonut("test", createTestNodeDiskMap(root))
 	c.Assert(err, IsNil)
 	// create bucket
-	err = donut.MakeBucket("foo")
+	err = donut.MakeBucket("foo", "private")
 	c.Assert(err, IsNil)
 
 	// check bucket exists
@@ -122,10 +122,10 @@ func (s *MySuite) TestMakeBucketWithSameNameFails(c *C) {
 	defer os.RemoveAll(root)
 	donut, err := NewDonut("test", createTestNodeDiskMap(root))
 	c.Assert(err, IsNil)
-	err = donut.MakeBucket("foo")
+	err = donut.MakeBucket("foo", "private")
 	c.Assert(err, IsNil)
 
-	err = donut.MakeBucket("foo")
+	err = donut.MakeBucket("foo", "private")
 	c.Assert(err, Not(IsNil))
 }
 
@@ -137,17 +137,17 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
 	donut, err := NewDonut("test", createTestNodeDiskMap(root))
 	c.Assert(err, IsNil)
 	// add a second bucket
-	err = donut.MakeBucket("foo")
+	err = donut.MakeBucket("foo", "private")
 	c.Assert(err, IsNil)
 
-	err = donut.MakeBucket("bar")
+	err = donut.MakeBucket("bar", "private")
 	c.Assert(err, IsNil)
 
 	buckets, err := donut.ListBuckets()
 	c.Assert(err, IsNil)
 	c.Assert(buckets, DeepEquals, []string{"bar", "foo"})
 
-	err = donut.MakeBucket("foobar")
+	err = donut.MakeBucket("foobar", "private")
 	c.Assert(err, IsNil)
 
 	buckets, err = donut.ListBuckets()
@@ -185,7 +185,7 @@ func (s *MySuite) TestNewObjectMetadata(c *C) {
 	expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil))
 	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
 
-	err = donut.MakeBucket("foo")
+	err = donut.MakeBucket("foo", "private")
 	c.Assert(err, IsNil)
 
 	err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
@@ -222,7 +222,7 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
 	donut, err := NewDonut("test", createTestNodeDiskMap(root))
 	c.Assert(err, IsNil)
 
-	err = donut.MakeBucket("foo")
+	err = donut.MakeBucket("foo", "private")
 	c.Assert(err, IsNil)
 
 	metadata := make(map[string]string)
@@ -263,7 +263,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
 	donut, err := NewDonut("test", createTestNodeDiskMap(root))
 	c.Assert(err, IsNil)
 
-	c.Assert(donut.MakeBucket("foo"), IsNil)
+	c.Assert(donut.MakeBucket("foo", "private"), IsNil)
 
 	one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
 	err = donut.PutObject("foo", "obj1", "", one, nil)

pkg/storage/donut/objectstorage.go

@@ -28,11 +28,11 @@ import (
 )
 
 // MakeBucket - make a new bucket
-func (d donut) MakeBucket(bucket string) error {
+func (d donut) MakeBucket(bucket, acl string) error {
 	if bucket == "" || strings.TrimSpace(bucket) == "" {
 		return iodine.New(errors.New("invalid argument"), nil)
 	}
-	return d.makeDonutBucket(bucket)
+	return d.makeDonutBucket(bucket, acl)
 }
 
 // GetBucketMetadata - get bucket metadata
@@ -47,6 +47,7 @@ func (d donut) GetBucketMetadata(bucket string) (map[string]string, error) {
 	metadata := make(map[string]string)
 	metadata["name"] = bucket
 	metadata["created"] = time.Now().Format(time.RFC3339Nano) // TODO get this, from whatever is written from SetBucketMetadata
+	metadata["acl"] = "private"
 	return metadata, nil
 }
 

pkg/storage/donut/objectstorage_internal.go

@@ -25,7 +25,8 @@ import (
 	"github.com/minio-io/minio/pkg/iodine"
 )
 
-func (d donut) makeDonutBucket(bucketName string) error {
+// TODO we have to store the acl's
+func (d donut) makeDonutBucket(bucketName, acl string) error {
 	err := d.getDonutBuckets()
 	if err != nil {
 		return iodine.New(err, nil)
@@ -33,7 +34,7 @@ func (d donut) makeDonutBucket(bucketName string) error {
 	if _, ok := d.buckets[bucketName]; ok {
 		return iodine.New(errors.New("bucket exists"), nil)
 	}
-	bucket, err := NewBucket(bucketName, d.name, d.nodes)
+	bucket, err := NewBucket(bucketName, acl, d.name, d.nodes)
 	if err != nil {
 		return iodine.New(err, nil)
 	}
@@ -74,7 +75,7 @@ func (d donut) getDonutBuckets() error {
 				}
 				bucketName := splitDir[0]
 				// we dont need this NewBucket once we cache from makeDonutBucket()
-				bucket, err := NewBucket(bucketName, d.name, d.nodes)
+				bucket, err := NewBucket(bucketName, "private", d.name, d.nodes)
 				if err != nil {
 					return iodine.New(err, nil)
 				}

pkg/storage/drivers/api_testsuite.go

@@ -48,14 +48,14 @@ func APITestSuite(c *check.C, create func() Driver) {
 
 func testCreateBucket(c *check.C, create func() Driver) {
 	drivers := create()
-	err := drivers.CreateBucket("bucket")
+	err := drivers.CreateBucket("bucket", "")
 	c.Assert(err, check.IsNil)
 }
 
 func testMultipleObjectCreation(c *check.C, create func() Driver) {
 	objects := make(map[string][]byte)
 	drivers := create()
-	err := drivers.CreateBucket("bucket")
+	err := drivers.CreateBucket("bucket", "")
 	c.Assert(err, check.IsNil)
 	for i := 0; i < 10; i++ {
 		randomPerm := rand.Perm(10)
@@ -94,7 +94,7 @@ func testMultipleObjectCreation(c *check.C, create func() Driver) {
 
 func testPaging(c *check.C, create func() Driver) {
 	drivers := create()
-	drivers.CreateBucket("bucket")
+	drivers.CreateBucket("bucket", "")
 	resources := BucketResourcesMetadata{}
 	objects, resources, err := drivers.ListObjects("bucket", resources)
 	c.Assert(err, check.IsNil)
@@ -198,7 +198,7 @@ func testPaging(c *check.C, create func() Driver) {
 
 func testObjectOverwriteFails(c *check.C, create func() Driver) {
 	drivers := create()
-	drivers.CreateBucket("bucket")
+	drivers.CreateBucket("bucket", "")
 
 	hasher1 := md5.New()
 	hasher1.Write([]byte("one"))
@@ -227,7 +227,7 @@ func testNonExistantBucketOperations(c *check.C, create func() Driver) {
 
 func testBucketMetadata(c *check.C, create func() Driver) {
 	drivers := create()
-	err := drivers.CreateBucket("string")
+	err := drivers.CreateBucket("string", "")
 	c.Assert(err, check.IsNil)
 
 	metadata, err := drivers.GetBucketMetadata("string")
@@ -237,15 +237,15 @@ func testBucketMetadata(c *check.C, create func() Driver) {
 
 func testBucketRecreateFails(c *check.C, create func() Driver) {
 	drivers := create()
-	err := drivers.CreateBucket("string")
+	err := drivers.CreateBucket("string", "")
 	c.Assert(err, check.IsNil)
-	err = drivers.CreateBucket("string")
+	err = drivers.CreateBucket("string", "")
 	c.Assert(err, check.Not(check.IsNil))
 }
 
 func testPutObjectInSubdir(c *check.C, create func() Driver) {
 	drivers := create()
-	err := drivers.CreateBucket("bucket")
+	err := drivers.CreateBucket("bucket", "")
 	c.Assert(err, check.IsNil)
 
 	hasher := md5.New()
@@ -270,7 +270,7 @@ func testListBuckets(c *check.C, create func() Driver) {
 	c.Assert(len(buckets), check.Equals, 0)
 
 	// add one and test exists
-	err = drivers.CreateBucket("bucket1")
+	err = drivers.CreateBucket("bucket1", "")
 	c.Assert(err, check.IsNil)
 
 	buckets, err = drivers.ListBuckets()
@@ -278,7 +278,7 @@ func testListBuckets(c *check.C, create func() Driver) {
 	c.Assert(err, check.IsNil)
 
 	// add two and test exists
-	err = drivers.CreateBucket("bucket2")
+	err = drivers.CreateBucket("bucket2", "")
 	c.Assert(err, check.IsNil)
 
 	buckets, err = drivers.ListBuckets()
@@ -286,7 +286,7 @@ func testListBuckets(c *check.C, create func() Driver) {
 	c.Assert(err, check.IsNil)
 
 	// add three and test exists + prefix
-	err = drivers.CreateBucket("bucket22")
+	err = drivers.CreateBucket("bucket22", "")
 
 	buckets, err = drivers.ListBuckets()
 	c.Assert(len(buckets), check.Equals, 3)
@@ -299,8 +299,8 @@ func testListBucketsOrder(c *check.C, create func() Driver) {
 	for i := 0; i < 10; i++ {
 		drivers := create()
 		// add one and test exists
-		drivers.CreateBucket("bucket1")
-		drivers.CreateBucket("bucket2")
+		drivers.CreateBucket("bucket1", "")
+		drivers.CreateBucket("bucket2", "")
 
 		buckets, err := drivers.ListBuckets()
 		c.Assert(len(buckets), check.Equals, 2)
@@ -321,7 +321,7 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Driver)
 
 func testNonExistantObjectInBucket(c *check.C, create func() Driver) {
 	drivers := create()
-	err := drivers.CreateBucket("bucket")
+	err := drivers.CreateBucket("bucket", "")
 	c.Assert(err, check.IsNil)
 
 	var byteBuffer bytes.Buffer
@@ -343,7 +343,7 @@ func testNonExistantObjectInBucket(c *check.C, create func() Driver) {
 
 func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
 	drivers := create()
-	err := drivers.CreateBucket("bucket")
+	err := drivers.CreateBucket("bucket", "")
 	c.Assert(err, check.IsNil)
 
 	err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
@@ -386,7 +386,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
 
 func testDefaultContentType(c *check.C, create func() Driver) {
 	drivers := create()
-	err := drivers.CreateBucket("bucket")
+	err := drivers.CreateBucket("bucket", "")
 	c.Assert(err, check.IsNil)
 
 	// test empty
@@ -408,10 +408,9 @@ func testDefaultContentType(c *check.C, create func() Driver) {
 	c.Assert(metadata.ContentType, check.Equals, "application/json")
 }
 
-/*
 func testContentMd5Set(c *check.C, create func() Driver) {
 	drivers := create()
-	err := drivers.CreateBucket("bucket")
+	err := drivers.CreateBucket("bucket", "")
 	c.Assert(err, check.IsNil)
 
 	// test md5 invalid
@@ -420,4 +419,3 @@ func testContentMd5Set(c *check.C, create func() Driver) {
 	err = drivers.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one"))
 	c.Assert(err, check.IsNil)
 }
-*/

pkg/storage/drivers/donut/donut.go

@@ -39,6 +39,7 @@ import (
 // donutDriver - creates a new single disk drivers driver using donut
 type donutDriver struct {
 	donut donut.Donut
+	path  string
 }
 
 const (
@@ -82,6 +83,7 @@ func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
 
 	s := new(donutDriver)
 	s.donut = donut
+	s.path = path
 
 	go start(ctrlChannel, errorChannel, s)
 	return ctrlChannel, errorChannel, s
@@ -117,11 +119,14 @@ func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error)
 }
 
 // CreateBucket creates a new bucket
-func (d donutDriver) CreateBucket(bucketName string) error {
-	if drivers.IsValidBucket(bucketName) && !strings.Contains(bucketName, ".") {
-		return d.donut.MakeBucket(bucketName)
+func (d donutDriver) CreateBucket(bucketName, acl string) error {
+	if !drivers.IsValidBucketACL(acl) {
+		return iodine.New(drivers.InvalidACL{ACL: acl}, nil)
 	}
-	return iodine.New(errors.New("Invalid bucket"), map[string]string{"bucket": bucketName})
+	if drivers.IsValidBucket(bucketName) && !strings.Contains(bucketName, ".") {
+		return d.donut.MakeBucket(bucketName, acl)
+	}
+	return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
 }
 
 // GetBucketMetadata retrieves an bucket's metadata
@@ -137,9 +142,14 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat
 	if err != nil {
 		return drivers.BucketMetadata{}, iodine.New(err, nil)
 	}
+	acl, ok := metadata["acl"]
+	if !ok {
+		return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{Path: d.path}, nil)
+	}
 	bucketMetadata := drivers.BucketMetadata{
 		Name:    metadata["name"],
 		Created: created,
+		ACL:     drivers.BucketACL(acl),
 	}
 	return bucketMetadata, nil
 }

pkg/storage/drivers/driver.go

@@ -27,7 +27,7 @@ import (
 type Driver interface {
 	// Bucket Operations
 	ListBuckets() ([]BucketMetadata, error)
-	CreateBucket(bucket string) error
+	CreateBucket(bucket, acl string) error
 	GetBucketMetadata(bucket string) (BucketMetadata, error)
 
 	// Object Operations
@@ -38,10 +38,40 @@ type Driver interface {
 	CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error
 }
 
+// BucketACL - bucket level access control
+type BucketACL string
+
+// different types of ACL's currently supported for buckets
+const (
+	BucketPrivate         = BucketACL("private")
+	BucketPublicRead      = BucketACL("public-read")
+	BucketPublicReadWrite = BucketACL("public-read-write")
+)
+
+func (b BucketACL) String() string {
+	return string(b)
+}
+
+// IsPrivate - is acl Private
+func (b BucketACL) IsPrivate() bool {
+	return b == BucketACL("private")
+}
+
+// IsPublicRead - is acl PublicRead
+func (b BucketACL) IsPublicRead() bool {
+	return b == BucketACL("public-read")
+}
+
+// IsPublicReadWrite - is acl PublicReadWrite
+func (b BucketACL) IsPublicReadWrite() bool {
+	return b == BucketACL("public-read-write")
+}
+
 // BucketMetadata - name and create date
 type BucketMetadata struct {
 	Name    string
 	Created time.Time
+	ACL     BucketACL
 }
 
 // ObjectMetadata - object key and its relevant metadata
@@ -98,6 +128,23 @@ func GetMode(resources BucketResourcesMetadata) FilterMode {
 	return f
 }
 
+// IsValidBucketACL - is provided acl string supported
+func IsValidBucketACL(acl string) bool {
+	switch acl {
+	case "private":
+		fallthrough
+	case "public-read":
+		fallthrough
+	case "public-read-write":
+		return true
+	case "":
+		// by default its "private"
+		return true
+	default:
+		return false
+	}
+}
+
 // IsDelimiterPrefixSet Delimiter and Prefix set
 func (b BucketResourcesMetadata) IsDelimiterPrefixSet() bool {
 	return b.Mode == DelimiterPrefixMode

pkg/storage/drivers/errors.go

@@ -54,6 +54,17 @@ type DigestError struct {
 	Md5    string
 }
 
+/// ACL related errors
+
+// InvalidACL - acl invalid
+type InvalidACL struct {
+	ACL string
+}
+
+func (e InvalidACL) Error() string {
+	return "Requested ACL is " + e.ACL + " invalid"
+}
+
 /// Bucket related errors
 
 // BucketNameInvalid - bucketname provided is invalid

pkg/storage/drivers/memory/memory.go

@@ -188,13 +188,16 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, md5sum string
 }
 
 // CreateBucket - create bucket in memory
-func (memory *memoryDriver) CreateBucket(bucketName string) error {
+func (memory *memoryDriver) CreateBucket(bucketName, acl string) error {
 	memory.lock.RLock()
 	if !drivers.IsValidBucket(bucketName) {
 		memory.lock.RUnlock()
 		return drivers.BucketNameInvalid{Bucket: bucketName}
 	}
-
+	if !drivers.IsValidBucketACL(acl) {
+		memory.lock.RUnlock()
+		return drivers.InvalidACL{ACL: acl}
+	}
 	if _, ok := memory.bucketMetadata[bucketName]; ok == true {
 		memory.lock.RUnlock()
 		return drivers.BucketExists{Bucket: bucketName}
@@ -205,6 +208,7 @@ func (memory *memoryDriver) CreateBucket(bucketName string) error {
 	newBucket.metadata = drivers.BucketMetadata{}
 	newBucket.metadata.Name = bucketName
 	newBucket.metadata.Created = time.Now()
+	newBucket.metadata.ACL = drivers.BucketACL(acl)
 	memory.lock.Lock()
 	defer memory.lock.Unlock()
 	memory.bucketMetadata[bucketName] = newBucket
@@ -234,7 +238,7 @@ func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedN
 	switch true {
 	case key == resources.Prefix:
 		keys = appendUniq(keys, key)
-		// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
+	// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
 	case key == resources.Prefix+delimitedName:
 		keys = appendUniq(keys, key)
 	case delimitedName != "":
@@ -339,8 +343,6 @@ func (memory *memoryDriver) evictObject(key lru.Key, value interface{}) {
 	k := key.(string)
 
 	memory.totalSize = memory.totalSize - memory.objectMetadata[k].metadata.Size
-
 	log.Println("evicting:", k)
-
 	delete(memory.objectMetadata, k)
 }

pkg/storage/drivers/mocks/Driver.go

@@ -27,8 +27,8 @@ func (m *Driver) ListBuckets() ([]drivers.BucketMetadata, error) {
 }
 
 // CreateBucket is a mock
-func (m *Driver) CreateBucket(bucket string) error {
-	ret := m.Called(bucket)
+func (m *Driver) CreateBucket(bucket, acl string) error {
+	ret := m.Called(bucket, acl)
 
 	r0 := ret.Error(0)