From 50dae0ab0400a25fcfbe7881c6cf70c4037694e3 Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Thu, 28 Jul 2016 20:49:08 -0700 Subject: [PATCH] bucket-policy: Migrate bucket policy to minioMetaBuket/buckets (#2321) --- bucket-handlers-listobjects.go | 4 +- bucket-handlers.go | 14 +++--- bucket-policy-handlers.go | 6 +-- bucket-policy-migrate.go | 83 ++++++++++++++++++++++++++++++++++ bucket-policy.go | 83 ++++++++++------------------------ object-handlers.go | 28 ++++++------ routers.go | 7 +++ 7 files changed, 141 insertions(+), 84 deletions(-) create mode 100644 bucket-policy-migrate.go diff --git a/bucket-handlers-listobjects.go b/bucket-handlers-listobjects.go index a6d1a1d43..ac868b53a 100644 --- a/bucket-handlers-listobjects.go +++ b/bucket-handlers-listobjects.go @@ -71,7 +71,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -130,7 +130,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } diff --git a/bucket-handlers.go b/bucket-handlers.go index ff964c475..878b5fbed 100644 --- a/bucket-handlers.go +++ b/bucket-handlers.go @@ -32,9 +32,9 @@ import ( ) // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html -func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) { +func enforceBucketPolicy(api objectAPIHandlers, action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) { // Read saved bucket policy. - policy, err := readBucketPolicy(bucket) + policy, err := readBucketPolicy(api, bucket) if err != nil { errorIf(err, "Unable read bucket policy.") switch err.(type) { @@ -84,7 +84,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -154,7 +154,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html - if s3Error := enforceBucketPolicy("s3:ListBucketMultipartUploads", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:ListBucketMultipartUploads", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -232,7 +232,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:DeleteObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -436,7 +436,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -474,7 +474,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http. } // Delete bucket access policy, if present - ignore any errors. - removeBucketPolicy(bucket) + removeBucketPolicy(api, bucket) // Write success response. writeSuccessNoContent(w) diff --git a/bucket-policy-handlers.go b/bucket-policy-handlers.go index 8e77b8ce9..fb190f01e 100644 --- a/bucket-policy-handlers.go +++ b/bucket-policy-handlers.go @@ -215,7 +215,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht } // Save bucket policy. - if err := writeBucketPolicy(bucket, bucketPolicyBuf); err != nil { + if err := writeBucketPolicy(api, bucket, bucketPolicyBuf); err != nil { errorIf(err, "Unable to write bucket policy.") switch err.(type) { case BucketNameInvalid: @@ -249,7 +249,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r } // Delete bucket access policy. - if err := removeBucketPolicy(bucket); err != nil { + if err := removeBucketPolicy(api, bucket); err != nil { errorIf(err, "Unable to remove bucket policy.") switch err.(type) { case BucketNameInvalid: @@ -285,7 +285,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht } // Read bucket access policy. - p, err := readBucketPolicy(bucket) + p, err := readBucketPolicy(api, bucket) if err != nil { errorIf(err, "Unable to read bucket policy.") switch err.(type) { diff --git a/bucket-policy-migrate.go b/bucket-policy-migrate.go new file mode 100644 index 000000000..99bd431a3 --- /dev/null +++ b/bucket-policy-migrate.go @@ -0,0 +1,83 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const policyJSON = "policy.json" + +func getBucketFromPolicyPath(oldPolicyPath string) string { + bucketPrefix, _ := filepath.Split(oldPolicyPath) + _, bucketName := filepath.Split(strings.TrimSuffix(bucketPrefix, slashSeparator)) + return bucketName +} + +func cleanupOldBucketPolicyConfigs() error { + // Get old bucket policy config directory. + oldBucketsConfigDir, err := getOldBucketsConfigPath() + fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy") + + // Recursively remove configDir/buckets/ - old bucket policy config location. + // N B This is called only if all bucket policies were successfully migrated. + return os.RemoveAll(oldBucketsConfigDir) +} + +func migrateBucketPolicyConfig(objAPI ObjectLayer) error { + // Get old bucket policy config directory. + oldBucketsConfigDir, err := getOldBucketsConfigPath() + fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy") + + // Check if config directory holding bucket policy exists before + // migration. + _, err = os.Stat(oldBucketsConfigDir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + // WalkFunc that migrates access-policy.json to + // .minio.sys/buckets/bucketName/policy.json on all disks. + migrateBucketPolicy := func(policyPath string, fileInfo os.FileInfo, err error) error { + // policyFile - e.g /configDir/sample-bucket/access-policy.json + if err != nil { + return err + } + // Skip entries that aren't bucket policy files. + if fileInfo.Name() != "access-policy.json" { + return nil + } + // Get bucketName from old policy file path. + bucketName := getBucketFromPolicyPath(policyPath) + // Read bucket policy config from old location. + policyBytes, err := ioutil.ReadFile(policyPath) + fatalIf(err, "Unable to read bucket policy to migrate bucket policy", policyPath) + newPolicyPath := retainSlash(bucketConfigPrefix) + retainSlash(bucketName) + policyJSON + // Erasure code the policy config to all the disks. + _, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), nil) + fatalIf(err, "Unable to write bucket policy during migration.", newPolicyPath) + return nil + } + return filepath.Walk(oldBucketsConfigDir, migrateBucketPolicy) +} diff --git a/bucket-policy.go b/bucket-policy.go index 325b12836..0c43cc478 100644 --- a/bucket-policy.go +++ b/bucket-policy.go @@ -17,13 +17,12 @@ package main import ( - "io/ioutil" - "os" + "bytes" "path/filepath" ) // getBucketsConfigPath - get buckets path. -func getBucketsConfigPath() (string, error) { +func getOldBucketsConfigPath() (string, error) { configPath, err := getConfigPath() if err != nil { return "", err @@ -32,97 +31,65 @@ func getBucketsConfigPath() (string, error) { } // getBucketConfigPath - get bucket config path. -func getBucketConfigPath(bucket string) (string, error) { - bucketsConfigPath, err := getBucketsConfigPath() +func getOldBucketConfigPath(bucket string) (string, error) { + bucketsConfigPath, err := getOldBucketsConfigPath() if err != nil { return "", err } return filepath.Join(bucketsConfigPath, bucket), nil } -// createBucketConfigPath - create bucket config directory. -func createBucketConfigPath(bucket string) error { - bucketConfigPath, err := getBucketConfigPath(bucket) - if err != nil { - return err - } - return os.MkdirAll(bucketConfigPath, 0700) -} - // readBucketPolicy - read bucket policy. -func readBucketPolicy(bucket string) ([]byte, error) { +func readBucketPolicy(api objectAPIHandlers, bucket string) ([]byte, error) { // Verify bucket is valid. if !IsValidBucketName(bucket) { return nil, BucketNameInvalid{Bucket: bucket} } - bucketConfigPath, err := getBucketConfigPath(bucket) + policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON) + objInfo, err := api.ObjectAPI.GetObjectInfo(minioMetaBucket, policyPath) if err != nil { - return nil, err - } - - // Get policy file. - bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") - if _, err = os.Stat(bucketPolicyFile); err != nil { - if os.IsNotExist(err) { + if _, ok := err.(ObjectNotFound); ok { return nil, BucketPolicyNotFound{Bucket: bucket} } return nil, err } - return ioutil.ReadFile(bucketPolicyFile) + var buffer bytes.Buffer + err = api.ObjectAPI.GetObject(minioMetaBucket, policyPath, 0, objInfo.Size, &buffer) + if err != nil { + if _, ok := err.(ObjectNotFound); ok { + return nil, BucketPolicyNotFound{Bucket: bucket} + } + return nil, err + } + return buffer.Bytes(), nil } // removeBucketPolicy - remove bucket policy. -func removeBucketPolicy(bucket string) error { +func removeBucketPolicy(api objectAPIHandlers, bucket string) error { // Verify bucket is valid. if !IsValidBucketName(bucket) { return BucketNameInvalid{Bucket: bucket} } - bucketConfigPath, err := getBucketConfigPath(bucket) - if err != nil { - return err - } - - // Get policy file. - bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") - if _, err = os.Stat(bucketPolicyFile); err != nil { - if os.IsNotExist(err) { + policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON) + if err := api.ObjectAPI.DeleteObject(minioMetaBucket, policyPath); err != nil { + if _, ok := err.(ObjectNotFound); ok { return BucketPolicyNotFound{Bucket: bucket} } return err } - if err := os.Remove(bucketPolicyFile); err != nil { - return err - } return nil } // writeBucketPolicy - save bucket policy. -func writeBucketPolicy(bucket string, accessPolicyBytes []byte) error { +func writeBucketPolicy(api objectAPIHandlers, bucket string, accessPolicyBytes []byte) error { // Verify if bucket path legal if !IsValidBucketName(bucket) { return BucketNameInvalid{Bucket: bucket} } - // Create bucket config path. - if err := createBucketConfigPath(bucket); err != nil { - return err - } - - bucketConfigPath, err := getBucketConfigPath(bucket) - if err != nil { - return err - } - - // Get policy file. - bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") - if _, err := os.Stat(bucketPolicyFile); err != nil { - if !os.IsNotExist(err) { - return err - } - } - - // Write bucket policy. - return ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600) + policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON) + _, err := api.ObjectAPI.PutObject(minioMetaBucket, policyPath, int64(len(accessPolicyBytes)), bytes.NewReader(accessPolicyBytes), nil) + return err } diff --git a/object-handlers.go b/object-handlers.go index 147c63a44..48e73aca5 100644 --- a/object-handlers.go +++ b/object-handlers.go @@ -53,13 +53,13 @@ func setGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { // this is in keeping with the permissions sections of the docs of both: // HEAD Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html // GET Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html -func errAllowableObjectNotFound(bucket string, r *http.Request) APIErrorCode { +func errAllowableObjectNotFound(api objectAPIHandlers, bucket string, r *http.Request) APIErrorCode { if getRequestAuthType(r) == authTypeAnonymous { //we care about the bucket as a whole, not a particular resource url := *r.URL url.Path = "/" + bucket - if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, &url); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, &url); s3Error != ErrNone { return ErrAccessDenied } } @@ -90,7 +90,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:GetObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:GetObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -106,7 +106,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req errorIf(err, "Unable to fetch object info.") apiErr := toAPIErrorCode(err) if apiErr == ErrNoSuchKey { - apiErr = errAllowableObjectNotFound(bucket, r) + apiErr = errAllowableObjectNotFound(api, bucket, r) } writeErrorResponse(w, r, apiErr, r.URL.Path) return @@ -196,7 +196,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:GetObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:GetObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -212,7 +212,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re errorIf(err, "Unable to fetch object info.") apiErr := toAPIErrorCode(err) if apiErr == ErrNoSuchKey { - apiErr = errAllowableObjectNotFound(bucket, r) + apiErr = errAllowableObjectNotFound(api, bucket, r) } writeErrorResponse(w, r, apiErr, r.URL.Path) return @@ -246,7 +246,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -418,7 +418,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -479,7 +479,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html - if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -558,7 +558,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html - if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -594,7 +594,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html - if s3Error := enforceBucketPolicy("s3:AbortMultipartUpload", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:AbortMultipartUpload", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -640,7 +640,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html - if s3Error := enforceBucketPolicy("s3:ListMultipartUploadParts", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:ListMultipartUploadParts", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -691,7 +691,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html - if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -815,7 +815,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html - if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone { + if s3Error := enforceBucketPolicy(api, "s3:DeleteObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } diff --git a/routers.go b/routers.go index ad615893e..b1a6b219a 100644 --- a/routers.go +++ b/routers.go @@ -45,6 +45,13 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler { objAPI, err := newObjectLayer(srvCmdConfig.disks, srvCmdConfig.ignoredDisks) fatalIf(err, "Unable to intialize object layer.") + // Migrate bucket policy from configDir to .minio.sys/buckets/ + err = migrateBucketPolicyConfig(objAPI) + fatalIf(err, "Unable to migrate bucket policy from config directory") + + err = cleanupOldBucketPolicyConfigs() + fatalIf(err, "Unable to clean up bucket policy from config directory.") + // Initialize storage rpc server. storageRPC, err := newRPCServer(srvCmdConfig.disks[0]) // FIXME: should only have one path. fatalIf(err, "Unable to initialize storage RPC server.")