bucket-policy: Migrate bucket policy to minioMetaBuket/buckets (#2321)

bucket-handlers-listobjects.go

@@ -71,7 +71,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -130,7 +130,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}

bucket-handlers.go

@@ -32,9 +32,9 @@ import (
 )
 
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) {
+func enforceBucketPolicy(api objectAPIHandlers, action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) {
 	// Read saved bucket policy.
-	policy, err := readBucketPolicy(bucket)
+	policy, err := readBucketPolicy(api, bucket)
 	if err != nil {
 		errorIf(err, "Unable read bucket policy.")
 		switch err.(type) {
@@ -84,7 +84,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -154,7 +154,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:ListBucketMultipartUploads", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:ListBucketMultipartUploads", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -232,7 +232,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -436,7 +436,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -474,7 +474,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Delete bucket access policy, if present - ignore any errors.
-	removeBucketPolicy(bucket)
+	removeBucketPolicy(api, bucket)
 
 	// Write success response.
 	writeSuccessNoContent(w)

bucket-policy-handlers.go

@@ -215,7 +215,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Save bucket policy.
-	if err := writeBucketPolicy(bucket, bucketPolicyBuf); err != nil {
+	if err := writeBucketPolicy(api, bucket, bucketPolicyBuf); err != nil {
 		errorIf(err, "Unable to write bucket policy.")
 		switch err.(type) {
 		case BucketNameInvalid:
@@ -249,7 +249,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Delete bucket access policy.
-	if err := removeBucketPolicy(bucket); err != nil {
+	if err := removeBucketPolicy(api, bucket); err != nil {
 		errorIf(err, "Unable to remove bucket policy.")
 		switch err.(type) {
 		case BucketNameInvalid:
@@ -285,7 +285,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Read bucket access policy.
-	p, err := readBucketPolicy(bucket)
+	p, err := readBucketPolicy(api, bucket)
 	if err != nil {
 		errorIf(err, "Unable to read bucket policy.")
 		switch err.(type) {

bucket-policy-migrate.go

@@ -0,0 +1,83 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const policyJSON = "policy.json"
+
+func getBucketFromPolicyPath(oldPolicyPath string) string {
+	bucketPrefix, _ := filepath.Split(oldPolicyPath)
+	_, bucketName := filepath.Split(strings.TrimSuffix(bucketPrefix, slashSeparator))
+	return bucketName
+}
+
+func cleanupOldBucketPolicyConfigs() error {
+	// Get old bucket policy config directory.
+	oldBucketsConfigDir, err := getOldBucketsConfigPath()
+	fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy")
+
+	// Recursively remove configDir/buckets/ - old bucket policy config location.
+	// N B This is called only if all bucket policies were successfully migrated.
+	return os.RemoveAll(oldBucketsConfigDir)
+}
+
+func migrateBucketPolicyConfig(objAPI ObjectLayer) error {
+	// Get old bucket policy config directory.
+	oldBucketsConfigDir, err := getOldBucketsConfigPath()
+	fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy")
+
+	// Check if config directory holding bucket policy exists before
+	// migration.
+	_, err = os.Stat(oldBucketsConfigDir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	// WalkFunc that migrates access-policy.json to
+	// .minio.sys/buckets/bucketName/policy.json on all disks.
+	migrateBucketPolicy := func(policyPath string, fileInfo os.FileInfo, err error) error {
+		// policyFile - e.g /configDir/sample-bucket/access-policy.json
+		if err != nil {
+			return err
+		}
+		// Skip entries that aren't bucket policy files.
+		if fileInfo.Name() != "access-policy.json" {
+			return nil
+		}
+		// Get bucketName from old policy file path.
+		bucketName := getBucketFromPolicyPath(policyPath)
+		// Read bucket policy config from old location.
+		policyBytes, err := ioutil.ReadFile(policyPath)
+		fatalIf(err, "Unable to read bucket policy to migrate bucket policy", policyPath)
+		newPolicyPath := retainSlash(bucketConfigPrefix) + retainSlash(bucketName) + policyJSON
+		// Erasure code the policy config to all the disks.
+		_, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), nil)
+		fatalIf(err, "Unable to write bucket policy during migration.", newPolicyPath)
+		return nil
+	}
+	return filepath.Walk(oldBucketsConfigDir, migrateBucketPolicy)
+}

bucket-policy.go

@@ -17,13 +17,12 @@
 package main
 
 import (
-	"io/ioutil"
+	"bytes"
-	"os"
 	"path/filepath"
 )
 
 // getBucketsConfigPath - get buckets path.
-func getBucketsConfigPath() (string, error) {
+func getOldBucketsConfigPath() (string, error) {
 	configPath, err := getConfigPath()
 	if err != nil {
 		return "", err
@@ -32,97 +31,65 @@ func getBucketsConfigPath() (string, error) {
 }
 
 // getBucketConfigPath - get bucket config path.
-func getBucketConfigPath(bucket string) (string, error) {
+func getOldBucketConfigPath(bucket string) (string, error) {
-	bucketsConfigPath, err := getBucketsConfigPath()
+	bucketsConfigPath, err := getOldBucketsConfigPath()
 	if err != nil {
 		return "", err
 	}
 	return filepath.Join(bucketsConfigPath, bucket), nil
 }
 
-// createBucketConfigPath - create bucket config directory.
-func createBucketConfigPath(bucket string) error {
-	bucketConfigPath, err := getBucketConfigPath(bucket)
-	if err != nil {
-		return err
-	}
-	return os.MkdirAll(bucketConfigPath, 0700)
-}
-
 // readBucketPolicy - read bucket policy.
-func readBucketPolicy(bucket string) ([]byte, error) {
+func readBucketPolicy(api objectAPIHandlers, bucket string) ([]byte, error) {
 	// Verify bucket is valid.
 	if !IsValidBucketName(bucket) {
 		return nil, BucketNameInvalid{Bucket: bucket}
 	}
 
-	bucketConfigPath, err := getBucketConfigPath(bucket)
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
+	objInfo, err := api.ObjectAPI.GetObjectInfo(minioMetaBucket, policyPath)
 	if err != nil {
-		return nil, err
+		if _, ok := err.(ObjectNotFound); ok {
-	}
-
-	// Get policy file.
-	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
-	if _, err = os.Stat(bucketPolicyFile); err != nil {
-		if os.IsNotExist(err) {
 			return nil, BucketPolicyNotFound{Bucket: bucket}
 		}
 		return nil, err
 	}
-	return ioutil.ReadFile(bucketPolicyFile)
+	var buffer bytes.Buffer
+	err = api.ObjectAPI.GetObject(minioMetaBucket, policyPath, 0, objInfo.Size, &buffer)
+	if err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return nil, BucketPolicyNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	return buffer.Bytes(), nil
 }
 
 // removeBucketPolicy - remove bucket policy.
-func removeBucketPolicy(bucket string) error {
+func removeBucketPolicy(api objectAPIHandlers, bucket string) error {
 	// Verify bucket is valid.
 	if !IsValidBucketName(bucket) {
 		return BucketNameInvalid{Bucket: bucket}
 	}
 
-	bucketConfigPath, err := getBucketConfigPath(bucket)
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
-	if err != nil {
+	if err := api.ObjectAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
-		return err
+		if _, ok := err.(ObjectNotFound); ok {
-	}
-
-	// Get policy file.
-	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
-	if _, err = os.Stat(bucketPolicyFile); err != nil {
-		if os.IsNotExist(err) {
 			return BucketPolicyNotFound{Bucket: bucket}
 		}
 		return err
 	}
-	if err := os.Remove(bucketPolicyFile); err != nil {
-		return err
-	}
 	return nil
 }
 
 // writeBucketPolicy - save bucket policy.
-func writeBucketPolicy(bucket string, accessPolicyBytes []byte) error {
+func writeBucketPolicy(api objectAPIHandlers, bucket string, accessPolicyBytes []byte) error {
 	// Verify if bucket path legal
 	if !IsValidBucketName(bucket) {
 		return BucketNameInvalid{Bucket: bucket}
 	}
 
-	// Create bucket config path.
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
-	if err := createBucketConfigPath(bucket); err != nil {
+	_, err := api.ObjectAPI.PutObject(minioMetaBucket, policyPath, int64(len(accessPolicyBytes)), bytes.NewReader(accessPolicyBytes), nil)
-		return err
+	return err
-	}
-
-	bucketConfigPath, err := getBucketConfigPath(bucket)
-	if err != nil {
-		return err
-	}
-
-	// Get policy file.
-	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
-	if _, err := os.Stat(bucketPolicyFile); err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-	}
-
-	// Write bucket policy.
-	return ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600)
 }

object-handlers.go

@@ -53,13 +53,13 @@ func setGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 // this is in keeping with the permissions sections of the docs of both:
 //   HEAD Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
 //   GET Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
-func errAllowableObjectNotFound(bucket string, r *http.Request) APIErrorCode {
+func errAllowableObjectNotFound(api objectAPIHandlers, bucket string, r *http.Request) APIErrorCode {
 	if getRequestAuthType(r) == authTypeAnonymous {
 		//we care about the bucket as a whole, not a particular resource
 		url := *r.URL
 		url.Path = "/" + bucket
 
-		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, &url); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, &url); s3Error != ErrNone {
 			return ErrAccessDenied
 		}
 	}
@@ -90,7 +90,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:GetObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:GetObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		errorIf(err, "Unable to fetch object info.")
 		apiErr := toAPIErrorCode(err)
 		if apiErr == ErrNoSuchKey {
-			apiErr = errAllowableObjectNotFound(bucket, r)
+			apiErr = errAllowableObjectNotFound(api, bucket, r)
 		}
 		writeErrorResponse(w, r, apiErr, r.URL.Path)
 		return
@@ -196,7 +196,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:GetObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:GetObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -212,7 +212,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		errorIf(err, "Unable to fetch object info.")
 		apiErr := toAPIErrorCode(err)
 		if apiErr == ErrNoSuchKey {
-			apiErr = errAllowableObjectNotFound(bucket, r)
+			apiErr = errAllowableObjectNotFound(api, bucket, r)
 		}
 		writeErrorResponse(w, r, apiErr, r.URL.Path)
 		return
@@ -246,7 +246,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -418,7 +418,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -479,7 +479,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -558,7 +558,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -594,7 +594,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:AbortMultipartUpload", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:AbortMultipartUpload", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -640,7 +640,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:ListMultipartUploadParts", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:ListMultipartUploadParts", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -691,7 +691,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -815,7 +815,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	case authTypeAnonymous:
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
+		if s3Error := enforceBucketPolicy(api, "s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}

routers.go

@@ -45,6 +45,13 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler {
 	objAPI, err := newObjectLayer(srvCmdConfig.disks, srvCmdConfig.ignoredDisks)
 	fatalIf(err, "Unable to intialize object layer.")
 
+	// Migrate bucket policy from configDir to .minio.sys/buckets/
+	err = migrateBucketPolicyConfig(objAPI)
+	fatalIf(err, "Unable to migrate bucket policy from config directory")
+
+	err = cleanupOldBucketPolicyConfigs()
+	fatalIf(err, "Unable to clean up bucket policy from config directory.")
+
 	// Initialize storage rpc server.
 	storageRPC, err := newRPCServer(srvCmdConfig.disks[0]) // FIXME: should only have one path.
 	fatalIf(err, "Unable to initialize storage RPC server.")