accessPolicy: Implement Put, Get, Delete access policy.

This patch implements Get,Put,Delete bucket policies

Supporting - http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html

Currently supports following actions.

   "*":                             true,
   "s3:*":                          true,
   "s3:GetObject":                  true,
   "s3:ListBucket":                 true,
   "s3:PutObject":                  true,
   "s3:CreateBucket":               true,
   "s3:GetBucketLocation":          true,
   "s3:DeleteBucket":               true,
   "s3:DeleteObject":               true,
   "s3:AbortMultipartUpload":       true,
   "s3:ListBucketMultipartUploads": true,
   "s3:ListMultipartUploadParts":   true,

following conditions for "StringEquals" and "StringNotEquals"

   "s3:prefix", "s3:max-keys"
This commit is contained in:
Harshavardhana
2016-02-03 16:46:56 -08:00
parent 846410c563
commit d5057b3c51
24 changed files with 1107 additions and 755 deletions

View File

@@ -38,7 +38,6 @@ func APITestSuite(c *check.C, create func() Filesystem) {
testPaging(c, create)
testObjectOverwriteWorks(c, create)
testNonExistantBucketOperations(c, create)
testBucketMetadata(c, create)
testBucketRecreateFails(c, create)
testPutObjectInSubdir(c, create)
testListBuckets(c, create)
@@ -53,13 +52,13 @@ func APITestSuite(c *check.C, create func() Filesystem) {
func testMakeBucket(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
}
func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
uploadID, err := fs.NewMultipartUpload("bucket", "key")
c.Assert(err, check.IsNil)
@@ -94,7 +93,7 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
uploadID, err := fs.NewMultipartUpload("bucket", "key")
c.Assert(err, check.IsNil)
@@ -126,7 +125,7 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
objects := make(map[string][]byte)
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
for i := 0; i < 10; i++ {
randomPerm := rand.Perm(10)
@@ -161,7 +160,7 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
func testPaging(c *check.C, create func() Filesystem) {
fs := create()
fs.MakeBucket("bucket", "")
fs.MakeBucket("bucket")
result, err := fs.ListObjects("bucket", "", "", "", 0)
c.Assert(err, check.IsNil)
c.Assert(len(result.Objects), check.Equals, 0)
@@ -262,7 +261,7 @@ func testPaging(c *check.C, create func() Filesystem) {
func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
hasher1 := md5.New()
@@ -292,27 +291,17 @@ func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
c.Assert(err, check.Not(check.IsNil))
}
func testBucketMetadata(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("string", "")
c.Assert(err, check.IsNil)
metadata, err := fs.GetBucketMetadata("string")
c.Assert(err, check.IsNil)
c.Assert(metadata.ACL, check.Equals, BucketACL("private"))
}
func testBucketRecreateFails(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("string", "")
err := fs.MakeBucket("string")
c.Assert(err, check.IsNil)
err = fs.MakeBucket("string", "")
err = fs.MakeBucket("string")
c.Assert(err, check.Not(check.IsNil))
}
func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
hasher := md5.New()
@@ -339,7 +328,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
c.Assert(len(buckets), check.Equals, 0)
// add one and test exists
err = fs.MakeBucket("bucket1", "")
err = fs.MakeBucket("bucket1")
c.Assert(err, check.IsNil)
buckets, err = fs.ListBuckets()
@@ -347,7 +336,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
c.Assert(err, check.IsNil)
// add two and test exists
err = fs.MakeBucket("bucket2", "")
err = fs.MakeBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err = fs.ListBuckets()
@@ -355,7 +344,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
c.Assert(err, check.IsNil)
// add three and test exists + prefix
err = fs.MakeBucket("bucket22", "")
err = fs.MakeBucket("bucket22")
buckets, err = fs.ListBuckets()
c.Assert(len(buckets), check.Equals, 3)
@@ -368,9 +357,9 @@ func testListBucketsOrder(c *check.C, create func() Filesystem) {
for i := 0; i < 10; i++ {
fs := create()
// add one and test exists
err := fs.MakeBucket("bucket1", "")
err := fs.MakeBucket("bucket1")
c.Assert(err, check.IsNil)
err = fs.MakeBucket("bucket2", "")
err = fs.MakeBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err := fs.ListBuckets()
c.Assert(err, check.IsNil)
@@ -390,7 +379,7 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesyst
func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
@@ -408,7 +397,7 @@ func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
@@ -443,7 +432,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem)
func testDefaultContentType(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
// test empty
@@ -455,7 +444,7 @@ func testDefaultContentType(c *check.C, create func() Filesystem) {
func testContentMD5Set(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
// test md5 invalid

View File

@@ -37,7 +37,6 @@ func APITestSuite(c *check.C, create func() Filesystem) {
testPaging(c, create)
testObjectOverwriteWorks(c, create)
testNonExistantBucketOperations(c, create)
testBucketMetadata(c, create)
testBucketRecreateFails(c, create)
testPutObjectInSubdir(c, create)
testListBuckets(c, create)
@@ -52,13 +51,13 @@ func APITestSuite(c *check.C, create func() Filesystem) {
func testMakeBucket(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
}
func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
uploadID, err := fs.NewMultipartUpload("bucket", "key")
c.Assert(err, check.IsNil)
@@ -93,7 +92,7 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
uploadID, err := fs.NewMultipartUpload("bucket", "key")
c.Assert(err, check.IsNil)
@@ -125,7 +124,7 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
objects := make(map[string][]byte)
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
for i := 0; i < 10; i++ {
randomPerm := rand.Perm(10)
@@ -160,7 +159,7 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
func testPaging(c *check.C, create func() Filesystem) {
fs := create()
fs.MakeBucket("bucket", "")
fs.MakeBucket("bucket")
result, err := fs.ListObjects("bucket", "", "", "", 0)
c.Assert(err, check.IsNil)
c.Assert(len(result.Objects), check.Equals, 0)
@@ -260,7 +259,7 @@ func testPaging(c *check.C, create func() Filesystem) {
func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
fs := create()
fs.MakeBucket("bucket", "")
fs.MakeBucket("bucket")
hasher1 := md5.New()
hasher1.Write([]byte("one"))
@@ -289,27 +288,17 @@ func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
c.Assert(err, check.Not(check.IsNil))
}
func testBucketMetadata(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("string", "private")
c.Assert(err, check.IsNil)
metadata, err := fs.GetBucketMetadata("string")
c.Assert(err, check.IsNil)
c.Assert(metadata.ACL, check.Equals, BucketACL("private"))
}
func testBucketRecreateFails(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("string", "private")
err := fs.MakeBucket("string")
c.Assert(err, check.IsNil)
err = fs.MakeBucket("string", "private")
err = fs.MakeBucket("string")
c.Assert(err, check.Not(check.IsNil))
}
func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "private")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
hasher := md5.New()
@@ -336,7 +325,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
c.Assert(len(buckets), check.Equals, 0)
// add one and test exists
err = fs.MakeBucket("bucket1", "")
err = fs.MakeBucket("bucket1")
c.Assert(err, check.IsNil)
buckets, err = fs.ListBuckets()
@@ -344,7 +333,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
c.Assert(err, check.IsNil)
// add two and test exists
err = fs.MakeBucket("bucket2", "")
err = fs.MakeBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err = fs.ListBuckets()
@@ -352,7 +341,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
c.Assert(err, check.IsNil)
// add three and test exists + prefix
err = fs.MakeBucket("bucket22", "")
err = fs.MakeBucket("bucket22")
buckets, err = fs.ListBuckets()
c.Assert(len(buckets), check.Equals, 3)
@@ -365,9 +354,9 @@ func testListBucketsOrder(c *check.C, create func() Filesystem) {
for i := 0; i < 10; i++ {
fs := create()
// add one and test exists
err := fs.MakeBucket("bucket1", "")
err := fs.MakeBucket("bucket1")
c.Assert(err, check.IsNil)
err = fs.MakeBucket("bucket2", "")
err = fs.MakeBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err := fs.ListBuckets()
c.Assert(err, check.IsNil)
@@ -387,7 +376,7 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesyst
func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
@@ -409,7 +398,7 @@ func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
@@ -444,7 +433,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem)
func testDefaultContentType(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
// test empty
@@ -456,7 +445,7 @@ func testDefaultContentType(c *check.C, create func() Filesystem) {
func testContentMD5Set(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket", "")
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
// test md5 invalid

View File

@@ -21,12 +21,7 @@ import (
"github.com/minio/minio/pkg/quick"
)
var multipartsMetadataPath, bucketsMetadataPath string
// setFSBucketsMetadataPath - set fs buckets metadata path.
func setFSBucketsMetadataPath(metadataPath string) {
bucketsMetadataPath = metadataPath
}
var multipartsMetadataPath string
// SetFSMultipartsMetadataPath - set custom multiparts session
// metadata path.
@@ -46,18 +41,6 @@ func saveMultipartsSession(multiparts Multiparts) *probe.Error {
return nil
}
// saveBucketsMetadata - save metadata of all buckets
func saveBucketsMetadata(buckets Buckets) *probe.Error {
qc, err := quick.New(buckets)
if err != nil {
return err.Trace()
}
if err := qc.Save(bucketsMetadataPath); err != nil {
return err.Trace()
}
return nil
}
// loadMultipartsSession load multipart session file
func loadMultipartsSession() (*Multiparts, *probe.Error) {
multiparts := &Multiparts{}
@@ -72,18 +55,3 @@ func loadMultipartsSession() (*Multiparts, *probe.Error) {
}
return qc.Data().(*Multiparts), nil
}
// loadBucketsMetadata load buckets metadata file
func loadBucketsMetadata() (*Buckets, *probe.Error) {
buckets := &Buckets{}
buckets.Version = "1"
buckets.Metadata = make(map[string]*BucketMetadata)
qc, err := quick.New(buckets)
if err != nil {
return nil, err.Trace()
}
if err := qc.Load(bucketsMetadataPath); err != nil {
return nil, err.Trace()
}
return qc.Data().(*Buckets), nil
}

View File

@@ -1,96 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package fs
// IsPrivateBucket - is private bucket
func (fs Filesystem) IsPrivateBucket(bucket string) bool {
fs.rwLock.RLock()
defer fs.rwLock.RUnlock()
bucketMetadata, ok := fs.buckets.Metadata[bucket]
if !ok {
return true
}
return bucketMetadata.ACL.IsPrivate()
}
// IsPublicBucket - is public bucket
func (fs Filesystem) IsPublicBucket(bucket string) bool {
fs.rwLock.RLock()
defer fs.rwLock.RUnlock()
bucketMetadata, ok := fs.buckets.Metadata[bucket]
if !ok {
return true
}
return bucketMetadata.ACL.IsPublicReadWrite()
}
// IsReadOnlyBucket - is read only bucket
func (fs Filesystem) IsReadOnlyBucket(bucket string) bool {
fs.rwLock.RLock()
defer fs.rwLock.RUnlock()
bucketMetadata, ok := fs.buckets.Metadata[bucket]
if !ok {
return true
}
return bucketMetadata.ACL.IsPublicRead()
}
// BucketACL - bucket level access control
type BucketACL string
// different types of ACL's currently supported for buckets
const (
BucketPrivate = BucketACL("private")
BucketPublicRead = BucketACL("public-read")
BucketPublicReadWrite = BucketACL("public-read-write")
)
func (b BucketACL) String() string {
return string(b)
}
// IsPrivate - is acl Private
func (b BucketACL) IsPrivate() bool {
return b == BucketACL("private")
}
// IsPublicRead - is acl PublicRead
func (b BucketACL) IsPublicRead() bool {
return b == BucketACL("public-read")
}
// IsPublicReadWrite - is acl PublicReadWrite
func (b BucketACL) IsPublicReadWrite() bool {
return b == BucketACL("public-read-write")
}
// IsValidBucketACL - is provided acl string supported
func IsValidBucketACL(acl string) bool {
switch acl {
case "private":
fallthrough
case "public-read":
fallthrough
case "public-read-write":
return true
case "":
// by default its "private"
return true
default:
return false
}
}

View File

@@ -52,15 +52,6 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
}
return probe.NewError(e)
}
// Critical region hold write lock.
fs.rwLock.Lock()
defer fs.rwLock.Unlock()
delete(fs.buckets.Metadata, bucket)
if err := saveBucketsMetadata(*fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil
}
@@ -112,8 +103,8 @@ func removeDuplicateBuckets(buckets []BucketMetadata) []BucketMetadata {
return buckets
}
// MakeBucket - PUT Bucket.
func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
// MakeBucket - PUT Bucket
func (fs Filesystem) MakeBucket(bucket string) *probe.Error {
di, err := disk.GetInfo(fs.path)
if err != nil {
return probe.NewError(err)
@@ -131,12 +122,6 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
// Verify if bucket acl is valid.
if !IsValidBucketACL(acl) {
return probe.NewError(InvalidACL{ACL: acl})
}
// Get bucket path.
bucket = fs.denormalizeBucket(bucket)
bucketDir := filepath.Join(fs.path, bucket)
if _, e := os.Stat(bucketDir); e == nil {
@@ -147,33 +132,6 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
if e := os.Mkdir(bucketDir, 0700); e != nil {
return probe.NewError(err)
}
fi, e := os.Stat(bucketDir)
// Check if bucket exists.
if e != nil {
if os.IsNotExist(e) {
return probe.NewError(BucketNotFound{Bucket: bucket})
}
return probe.NewError(e)
}
if acl == "" {
acl = "private"
}
// Get a new bucket name metadata.
bucketMetadata := &BucketMetadata{}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
bucketMetadata.ACL = BucketACL(acl)
// Critical region hold a write lock.
fs.rwLock.Lock()
defer fs.rwLock.Unlock()
fs.buckets.Metadata[bucket] = bucketMetadata
if err := saveBucketsMetadata(*fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil
}
@@ -200,7 +158,6 @@ func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Er
if !IsValidBucketName(bucket) {
return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
bucket = fs.denormalizeBucket(bucket)
// Get bucket path.
bucketDir := filepath.Join(fs.path, bucket)
@@ -212,44 +169,8 @@ func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Er
}
return BucketMetadata{}, probe.NewError(e)
}
fs.rwLock.RLock()
bucketMetadata, ok := fs.buckets.Metadata[bucket]
fs.rwLock.RUnlock()
// If metadata value is not found, get it from disk.
if !ok {
bucketMetadata = &BucketMetadata{}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
bucketMetadata.ACL = BucketACL("private")
}
return *bucketMetadata, nil
}
// SetBucketMetadata - set bucket metadata.
func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error {
bucketMetadata, err := fs.GetBucketMetadata(bucket)
if err != nil {
return err
}
// Save the acl.
acl := metadata["acl"]
if !IsValidBucketACL(acl) {
return probe.NewError(InvalidACL{ACL: acl})
} else if acl == "" {
acl = "private"
}
bucketMetadata.ACL = BucketACL(acl)
bucket = fs.denormalizeBucket(bucket)
// Critical region handle write lock.
fs.rwLock.Lock()
defer fs.rwLock.Unlock()
fs.buckets.Metadata[bucket] = &bucketMetadata
if err := saveBucketsMetadata(*fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil
bucketMetadata := BucketMetadata{}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
return bucketMetadata, nil
}

View File

@@ -65,7 +65,7 @@ func BenchmarkDeleteBucket(b *testing.B) {
b.StopTimer()
// Create and delete the bucket over and over.
err = filesystem.MakeBucket("bucket", "public-read-write")
err = filesystem.MakeBucket("bucket")
if err != nil {
b.Fatal(err)
}
@@ -94,7 +94,7 @@ func BenchmarkGetBucketMetadata(b *testing.B) {
}
// Put up a bucket with some metadata.
err = filesystem.MakeBucket("bucket", "public-read-write")
err = filesystem.MakeBucket("bucket")
if err != nil {
b.Fatal(err)
}
@@ -109,37 +109,3 @@ func BenchmarkGetBucketMetadata(b *testing.B) {
}
}
}
func BenchmarkSetBucketMetadata(b *testing.B) {
// Make a temporary directory to use as the filesystem.
directory, fserr := ioutil.TempDir("", "minio-benchmark")
if fserr != nil {
b.Fatal(fserr)
}
defer os.RemoveAll(directory)
// Create the filesystem.
filesystem, err := New(directory, 0)
if err != nil {
b.Fatal(err)
}
// Put up a bucket with some metadata.
err = filesystem.MakeBucket("bucket", "public-read-write")
if err != nil {
b.Fatal(err)
}
metadata := make(map[string]string)
metadata["acl"] = "public-read-write"
b.ResetTimer()
for i := 0; i < b.N; i++ {
// Set all the metadata!
err = filesystem.SetBucketMetadata("bucket", metadata)
if err != nil {
b.Fatal(err)
}
}
}

View File

@@ -25,7 +25,6 @@ import (
type BucketMetadata struct {
Name string
Created time.Time
ACL BucketACL
}
// ObjectMetadata - object key and its relevant metadata

View File

@@ -163,6 +163,13 @@ type GenericBucketError struct {
Bucket string
}
// BucketPolicyNotFound - no bucket policy found.
type BucketPolicyNotFound GenericBucketError
func (e BucketPolicyNotFound) Error() string {
return "No bucket policy found for bucket: " + e.Bucket
}
// GenericObjectError - generic object error
type GenericObjectError struct {
Bucket string
@@ -183,17 +190,6 @@ type DigestError struct {
MD5 string
}
/// ACL related errors
// InvalidACL - acl invalid
type InvalidACL struct {
ACL string
}
func (e InvalidACL) Error() string {
return "Requested ACL is " + e.ACL + " invalid"
}
/// Bucket related errors
// BucketNameInvalid - bucketname provided is invalid

View File

@@ -31,17 +31,10 @@ type Filesystem struct {
minFreeDisk int64
rwLock *sync.RWMutex
multiparts *Multiparts
buckets *Buckets
listServiceReqCh chan<- listServiceReq
timeoutReqCh chan<- uint32
}
// Buckets holds acl information
type Buckets struct {
Version string `json:"version"`
Metadata map[string]*BucketMetadata
}
// MultipartSession holds active session information
type MultipartSession struct {
TotalParts int
@@ -59,7 +52,6 @@ type Multiparts struct {
// New instantiate a new donut
func New(rootPath string, minFreeDisk int64) (Filesystem, *probe.Error) {
setFSBucketsMetadataPath(filepath.Join(rootPath, "$buckets.json"))
setFSMultipartsMetadataPath(filepath.Join(rootPath, "$multiparts-session.json"))
var err *probe.Error
@@ -80,27 +72,12 @@ func New(rootPath string, minFreeDisk int64) (Filesystem, *probe.Error) {
}
}
var buckets *Buckets
buckets, err = loadBucketsMetadata()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
buckets = &Buckets{
Version: "1",
Metadata: make(map[string]*BucketMetadata),
}
if err = saveBucketsMetadata(*buckets); err != nil {
return Filesystem{}, err.Trace()
}
} else {
return Filesystem{}, err.Trace()
}
}
fs := Filesystem{
rwLock: &sync.RWMutex{},
}
fs.path = rootPath
fs.multiparts = multiparts
fs.buckets = buckets
/// Defaults
// minium free disk required for i/o operations to succeed.

33
pkg/s3/access/README.md Normal file
View File

@@ -0,0 +1,33 @@
## Access Policy
This package implements parsing and validating bucket access policies based on Access Policy Language specification - http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
### Supports following effects.
Allow
Deny
### Supports following set of operations.
*
s3:*
s3:GetObject
s3:ListBucket
s3:PutObject
s3:CreateBucket
s3:GetBucketLocation
s3:DeleteBucket
s3:DeleteObject
s3:AbortMultipartUpload
s3:ListBucketMultipartUploads
s3:ListMultipartUploadParts
### Supports following conditions.
StringEquals
StringNotEquals
Supported applicable condition keys for each conditions.
s3:prefix
s3:max-keys

230
pkg/s3/access/policy.go Normal file
View File

@@ -0,0 +1,230 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Package accesspolicy implements AWS Access Policy Language parser in
// accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
package accesspolicy
import (
"encoding/json"
"errors"
"fmt"
"strings"
)
const (
// AWSResourcePrefix - bucket policy resource prefix.
AWSResourcePrefix = "arn:aws:s3:::"
)
// supportedActionMap - lists all the actions supported by minio.
var supportedActionMap = map[string]struct{}{
"*": {},
"s3:*": {},
"s3:GetObject": {},
"s3:ListBucket": {},
"s3:PutObject": {},
"s3:CreateBucket": {},
"s3:GetBucketLocation": {},
"s3:DeleteBucket": {},
"s3:DeleteObject": {},
"s3:AbortMultipartUpload": {},
"s3:ListBucketMultipartUploads": {},
"s3:ListMultipartUploadParts": {},
}
// User - canonical users list.
type User struct {
AWS []string
}
// Statement - minio policy statement
type Statement struct {
Sid string
Effect string
Principal User
Actions []string `json:"Action"`
Resources []string `json:"Resource"`
Conditions map[string]map[string]string `json:"Condition"`
}
// BucketPolicy - minio policy collection
type BucketPolicy struct {
Version string // date in 0000-00-00 format
Statements []Statement `json:"Statement"`
}
// supportedEffectMap - supported effects.
var supportedEffectMap = map[string]struct{}{
"Allow": {},
"Deny": {},
}
// isValidActions - are actions valid.
func isValidActions(actions []string) (err error) {
// Statement actions cannot be empty.
if len(actions) == 0 {
err = errors.New("Action list cannot be empty.")
return err
}
for _, action := range actions {
if _, ok := supportedActionMap[action]; !ok {
err = errors.New("Unsupported action found: " + action + ", please validate your policy document.")
return err
}
}
return nil
}
// isValidEffect - is effect valid.
func isValidEffect(effect string) error {
// Statement effect cannot be empty.
if len(effect) == 0 {
err := errors.New("Policy effect cannot be empty.")
return err
}
_, ok := supportedEffectMap[effect]
if !ok {
err := errors.New("Unsupported Effect found: " + effect + ", please validate your policy document.")
return err
}
return nil
}
// isValidResources - are valid resources.
func isValidResources(resources []string) (err error) {
// Statement resources cannot be empty.
if len(resources) == 0 {
err = errors.New("Resource list cannot be empty.")
return err
}
for _, resource := range resources {
if !strings.HasPrefix(resource, AWSResourcePrefix) {
err = errors.New("Unsupported resource style found: " + resource + ", please validate your policy document.")
return err
}
resourceSuffix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
if len(resourceSuffix) == 0 || strings.HasPrefix(resourceSuffix, "/") {
err = errors.New("Invalid resource style found: " + resource + ", please validate your policy document.")
return err
}
}
return nil
}
// isValidPrincipals - are valid principals.
func isValidPrincipals(principals []string) (err error) {
// Statement principal should have a value.
if len(principals) == 0 {
err = errors.New("Principal cannot be empty.")
return err
}
var ok bool
for _, principal := range principals {
// Minio does not support or implement IAM, "*" is the only valid value.
if principal == "*" {
ok = true
continue
}
ok = false
}
if !ok {
err = errors.New("Unsupported principal style found: " + strings.Join(principals, " ") + ", please validate your policy document.")
return err
}
return nil
}
func isValidConditions(conditions map[string]map[string]string) (err error) {
// Verify conditions should be valid.
if len(conditions) > 0 {
// Validate if stringEquals, stringNotEquals are present
// if not throw an error.
_, stringEqualsOK := conditions["StringEquals"]
_, stringNotEqualsOK := conditions["StringNotEquals"]
if !stringEqualsOK && !stringNotEqualsOK {
err = fmt.Errorf("Unsupported condition type found: %s, please validate your policy document.", conditions)
return err
}
// Validate s3:prefix, s3:max-keys are present if not
// throw an error.
if len(conditions["StringEquals"]) > 0 {
_, s3PrefixOK := conditions["StringEquals"]["s3:prefix"]
_, s3MaxKeysOK := conditions["StringEquals"]["s3:max-keys"]
if !s3PrefixOK && !s3MaxKeysOK {
err = fmt.Errorf("Unsupported condition keys found: %s, please validate your policy document.",
conditions["StringEquals"])
return err
}
}
if len(conditions["StringNotEquals"]) > 0 {
_, s3PrefixOK := conditions["StringNotEquals"]["s3:prefix"]
_, s3MaxKeysOK := conditions["StringNotEquals"]["s3:max-keys"]
if !s3PrefixOK && !s3MaxKeysOK {
err = fmt.Errorf("Unsupported condition keys found: %s, please validate your policy document.",
conditions["StringNotEquals"])
return err
}
}
}
return nil
}
// Validate - validate if request body is of proper JSON and in
// accordance with policy standards.
func Validate(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
if err = json.Unmarshal(bucketPolicyBuf, &policy); err != nil {
return BucketPolicy{}, err
}
// Policy version cannot be empty.
if len(policy.Version) == 0 {
err = errors.New("Policy version cannot be empty.")
return BucketPolicy{}, err
}
// Policy statements cannot be empty.
if len(policy.Statements) == 0 {
err = errors.New("Policy statement cannot be empty.")
return BucketPolicy{}, err
}
// Loop through all policy statements and validate entries.
for _, statement := range policy.Statements {
// Statement effect should be valid.
if err := isValidEffect(statement.Effect); err != nil {
return BucketPolicy{}, err
}
// Statement principal should be supported format.
if err := isValidPrincipals(statement.Principal.AWS); err != nil {
return BucketPolicy{}, err
}
// Statement actions should be valid.
if err := isValidActions(statement.Actions); err != nil {
return BucketPolicy{}, err
}
// Statment resources should be valid.
if err := isValidResources(statement.Resources); err != nil {
return BucketPolicy{}, err
}
// Statement conditions should be valid.
if err := isValidConditions(statement.Conditions); err != nil {
return BucketPolicy{}, err
}
}
// Return successfully parsed policy structure.
return policy, nil
}