Enhance policy handling to support SSE and WORM (#5790)

- remove old bucket policy handling
- add new policy handling
- add new policy handling unit tests

This patch brings support to bucket policy to have more control not
limiting to anonymous.  Bucket owner controls to allow/deny any rest
API.

For example server side encryption can be controlled by allowing
PUT/GET objects with encryptions including bucket owner.
This commit is contained in:
Bala FA
2018-04-25 04:23:30 +05:30
committed by kannappanr
parent 21a3c0f482
commit 0d52126023
77 changed files with 9811 additions and 2633 deletions

View File

@@ -34,10 +34,12 @@ import (
"github.com/Azure/azure-sdk-for-go/storage"
humanize "github.com/dustin/go-humanize"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
miniogopolicy "github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/policy"
"github.com/minio/minio/pkg/policy/condition"
sha256 "github.com/minio/sha256-simd"
minio "github.com/minio/minio/cmd"
@@ -1020,10 +1022,16 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
// storage.ContainerAccessTypePrivate - none in minio terminology
// As the common denominator for minio and azure is readonly and none, we support
// these two policies at the bucket level.
func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
var policies []minio.BucketAccessPolicy
func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
if err != nil {
// This should not happen.
logger.LogIf(ctx, err)
return azureToObjectError(err, bucket)
}
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") {
var policies []minio.BucketAccessPolicy
for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
policies = append(policies, minio.BucketAccessPolicy{
Prefix: prefix,
Policy: policy,
@@ -1038,7 +1046,7 @@ func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, polic
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
if policies[0].Policy != policy.BucketPolicyReadOnly {
if policies[0].Policy != miniogopolicy.BucketPolicyReadOnly {
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
@@ -1047,31 +1055,47 @@ func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, polic
AccessPolicies: nil,
}
container := a.client.GetContainerReference(bucket)
err := container.SetPermissions(perm, nil)
err = container.SetPermissions(perm, nil)
logger.LogIf(ctx, err)
return azureToObjectError(err, bucket)
}
// GetBucketPolicy - Get the container ACL and convert it to canonical []bucketAccessPolicy
func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
container := a.client.GetContainerReference(bucket)
perm, err := container.GetPermissions(nil)
if err != nil {
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, azureToObjectError(err, bucket)
return nil, azureToObjectError(err, bucket)
}
switch perm.AccessType {
case storage.ContainerAccessTypePrivate:
logger.LogIf(ctx, minio.PolicyNotFound{Bucket: bucket})
return policy.BucketAccessPolicy{}, minio.PolicyNotFound{Bucket: bucket}
case storage.ContainerAccessTypeContainer:
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
default:
if perm.AccessType == storage.ContainerAccessTypePrivate {
logger.LogIf(ctx, minio.BucketPolicyNotFound{Bucket: bucket})
return nil, minio.BucketPolicyNotFound{Bucket: bucket}
} else if perm.AccessType != storage.ContainerAccessTypeContainer {
logger.LogIf(ctx, minio.NotImplemented{})
return policy.BucketAccessPolicy{}, azureToObjectError(minio.NotImplemented{})
return nil, azureToObjectError(minio.NotImplemented{})
}
return policyInfo, nil
return &policy.Policy{
Version: policy.DefaultVersion,
Statements: []policy.Statement{
policy.NewStatement(
policy.Allow,
policy.NewPrincipal("*"),
policy.NewActionSet(
policy.GetBucketLocationAction,
policy.ListBucketAction,
policy.GetObjectAction,
),
policy.NewResourceSet(
policy.NewResource(bucket, ""),
policy.NewResource(bucket, "*"),
),
condition.NewFunctions(),
),
},
}, nil
}
// DeleteBucketPolicy - Set the container ACL to "private"

View File

@@ -30,10 +30,12 @@ import (
b2 "github.com/minio/blazer/base"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
miniogopolicy "github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
h2 "github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/policy"
"github.com/minio/minio/pkg/policy/condition"
minio "github.com/minio/minio/cmd"
)
@@ -722,10 +724,15 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
// bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
// bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
// Default is AllPrivate for all buckets.
func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
var policies []minio.BucketAccessPolicy
func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
if err != nil {
// This should not happen.
return b2ToObjectError(err, bucket)
}
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") {
var policies []minio.BucketAccessPolicy
for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
policies = append(policies, minio.BucketAccessPolicy{
Prefix: prefix,
Policy: policy,
@@ -740,7 +747,7 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
if policies[0].Policy != policy.BucketPolicyReadOnly {
if policies[0].Policy != miniogopolicy.BucketPolicyReadOnly {
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
@@ -756,21 +763,39 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn
// GetBucketPolicy, returns the current bucketType from B2 backend and convert
// it into S3 compatible bucket policy info.
func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
bkt, err := l.Bucket(ctx, bucket)
if err != nil {
return policyInfo, err
}
if bkt.Type == bucketTypeReadOnly {
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
return policyInfo, nil
return nil, err
}
// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
// just return back as policy not found for all cases.
// CreateBucket always sets the value to allPrivate by default.
logger.LogIf(ctx, minio.PolicyNotFound{Bucket: bucket})
return policy.BucketAccessPolicy{}, minio.PolicyNotFound{Bucket: bucket}
if bkt.Type != bucketTypeReadOnly {
logger.LogIf(ctx, minio.BucketPolicyNotFound{Bucket: bucket})
return nil, minio.BucketPolicyNotFound{Bucket: bucket}
}
return &policy.Policy{
Version: policy.DefaultVersion,
Statements: []policy.Statement{
policy.NewStatement(
policy.Allow,
policy.NewPrincipal("*"),
policy.NewActionSet(
policy.GetBucketLocationAction,
policy.ListBucketAction,
policy.GetObjectAction,
),
policy.NewResourceSet(
policy.NewResource(bucket, ""),
policy.NewResource(bucket, "*"),
),
condition.NewFunctions(),
),
},
}, nil
}
// DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'.

View File

@@ -33,10 +33,12 @@ import (
"cloud.google.com/go/storage"
humanize "github.com/dustin/go-humanize"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
miniogopolicy "github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/policy"
"github.com/minio/minio/pkg/policy/condition"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
@@ -1132,10 +1134,16 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
}
// SetBucketPolicy - Set policy on bucket
func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
var policies []minio.BucketAccessPolicy
func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
if err != nil {
// This should not happen.
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket)
}
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") {
var policies []minio.BucketAccessPolicy
for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
policies = append(policies, minio.BucketAccessPolicy{
Prefix: prefix,
Policy: policy,
@@ -1154,7 +1162,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
}
acl := l.client.Bucket(bucket).ACL()
if policies[0].Policy == policy.BucketPolicyNone {
if policies[0].Policy == miniogopolicy.BucketPolicyNone {
if err := acl.Delete(l.ctx, storage.AllUsers); err != nil {
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket)
@@ -1164,9 +1172,9 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
var role storage.ACLRole
switch policies[0].Policy {
case policy.BucketPolicyReadOnly:
case miniogopolicy.BucketPolicyReadOnly:
role = storage.RoleReader
case policy.BucketPolicyWriteOnly:
case miniogopolicy.BucketPolicyWriteOnly:
role = storage.RoleWriter
default:
logger.LogIf(ctx, minio.NotImplemented{})
@@ -1182,30 +1190,63 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
}
// GetBucketPolicy - Get policy on bucket
func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
if err != nil {
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, gcsToObjectError(err, bucket)
return nil, gcsToObjectError(err, bucket)
}
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
var readOnly, writeOnly bool
for _, r := range rules {
if r.Entity != storage.AllUsers || r.Role == storage.RoleOwner {
continue
}
switch r.Role {
case storage.RoleReader:
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
readOnly = true
case storage.RoleWriter:
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyWriteOnly, bucket, "")
writeOnly = true
}
}
// Return NoSuchBucketPolicy error, when policy is not set
if len(policyInfo.Statements) == 0 {
logger.LogIf(ctx, minio.PolicyNotFound{})
return policy.BucketAccessPolicy{}, gcsToObjectError(minio.PolicyNotFound{}, bucket)
actionSet := policy.NewActionSet()
if readOnly {
actionSet.Add(policy.GetBucketLocationAction)
actionSet.Add(policy.ListBucketAction)
actionSet.Add(policy.GetObjectAction)
}
return policyInfo, nil
if writeOnly {
actionSet.Add(policy.GetBucketLocationAction)
actionSet.Add(policy.ListBucketMultipartUploadsAction)
actionSet.Add(policy.AbortMultipartUploadAction)
actionSet.Add(policy.DeleteObjectAction)
actionSet.Add(policy.ListMultipartUploadPartsAction)
actionSet.Add(policy.PutObjectAction)
}
// Return NoSuchBucketPolicy error, when policy is not set
if len(actionSet) == 0 {
logger.LogIf(ctx, minio.BucketPolicyNotFound{})
return nil, gcsToObjectError(minio.BucketPolicyNotFound{}, bucket)
}
return &policy.Policy{
Version: policy.DefaultVersion,
Statements: []policy.Statement{
policy.NewStatement(
policy.Allow,
policy.NewPrincipal("*"),
actionSet,
policy.NewResourceSet(
policy.NewResource(bucket, ""),
policy.NewResource(bucket, "*"),
),
condition.NewFunctions(),
),
},
}, nil
}
// DeleteBucketPolicy - Delete all policies on bucket

View File

@@ -20,9 +20,9 @@ import (
"context"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
minio "github.com/minio/minio/cmd"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/policy"
)
const (
@@ -132,6 +132,6 @@ func (l *nasObjects) IsNotificationSupported() bool {
}
// GetBucketPolicy will get policy on bucket
func (l *nasObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
return minio.ReadBucketPolicy(bucket, l)
func (l *nasObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
return minio.GetPolicyConfig(l, bucket)
}

View File

@@ -30,11 +30,13 @@ import (
"github.com/dustin/go-humanize"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
miniogopolicy "github.com/minio/minio-go/pkg/policy"
minio "github.com/minio/minio/cmd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/policy"
"github.com/minio/minio/pkg/policy/condition"
)
const (
@@ -962,8 +964,14 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
// oss.ACLPublicReadWrite: readwrite in minio terminology
// oss.ACLPublicRead: readonly in minio terminology
// oss.ACLPrivate: none in minio terminology
func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
bucketPolicies := policy.GetPolicies(policyInfo.Statements, bucket, "")
func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
if err != nil {
// This should not happen.
return ossToObjectError(err, bucket)
}
bucketPolicies := miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "")
if len(bucketPolicies) != 1 {
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
@@ -978,11 +986,11 @@ func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyI
var acl oss.ACLType
switch bucketPolicy {
case policy.BucketPolicyNone:
case miniogopolicy.BucketPolicyNone:
acl = oss.ACLPrivate
case policy.BucketPolicyReadOnly:
case miniogopolicy.BucketPolicyReadOnly:
acl = oss.ACLPublicRead
case policy.BucketPolicyReadWrite:
case miniogopolicy.BucketPolicyReadWrite:
acl = oss.ACLPublicReadWrite
default:
logger.LogIf(ctx, minio.NotImplemented{})
@@ -1000,29 +1008,60 @@ func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyI
}
// GetBucketPolicy will get policy on bucket.
func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
result, err := l.Client.GetBucketACL(bucket)
if err != nil {
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, ossToObjectError(err)
return nil, ossToObjectError(err)
}
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
var readOnly, readWrite bool
switch result.ACL {
case string(oss.ACLPrivate):
// By default, all buckets starts with a "private" policy.
logger.LogIf(ctx, minio.PolicyNotFound{})
return policy.BucketAccessPolicy{}, ossToObjectError(minio.PolicyNotFound{}, bucket)
logger.LogIf(ctx, minio.BucketPolicyNotFound{})
return nil, ossToObjectError(minio.BucketPolicyNotFound{}, bucket)
case string(oss.ACLPublicRead):
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
readOnly = true
case string(oss.ACLPublicReadWrite):
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadWrite, bucket, "")
readWrite = true
default:
logger.LogIf(ctx, minio.NotImplemented{})
return policy.BucketAccessPolicy{}, minio.NotImplemented{}
return nil, minio.NotImplemented{}
}
return policyInfo, nil
actionSet := policy.NewActionSet()
if readOnly {
actionSet.Add(policy.GetBucketLocationAction)
actionSet.Add(policy.ListBucketAction)
actionSet.Add(policy.GetObjectAction)
}
if readWrite {
actionSet.Add(policy.GetBucketLocationAction)
actionSet.Add(policy.ListBucketAction)
actionSet.Add(policy.GetObjectAction)
actionSet.Add(policy.ListBucketMultipartUploadsAction)
actionSet.Add(policy.AbortMultipartUploadAction)
actionSet.Add(policy.DeleteObjectAction)
actionSet.Add(policy.ListMultipartUploadPartsAction)
actionSet.Add(policy.PutObjectAction)
}
return &policy.Policy{
Version: policy.DefaultVersion,
Statements: []policy.Statement{
policy.NewStatement(
policy.Allow,
policy.NewPrincipal("*"),
actionSet,
policy.NewResourceSet(
policy.NewResource(bucket, ""),
policy.NewResource(bucket, "*"),
),
condition.NewFunctions(),
),
},
}, nil
}
// DeleteBucketPolicy deletes all policies on bucket.

View File

@@ -20,14 +20,15 @@ import (
"context"
"encoding/json"
"io"
"strings"
"github.com/minio/cli"
miniogo "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio-go/pkg/s3utils"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/policy"
minio "github.com/minio/minio/cmd"
)
@@ -428,31 +429,32 @@ func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
}
// SetBucketPolicy sets policy on bucket
func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
data, err := json.Marshal(&policyInfo)
func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
data, err := json.Marshal(bucketPolicy)
if err != nil {
return err
// This should not happen.
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket)
}
if err := l.Client.SetBucketPolicy(bucket, string(data)); err != nil {
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, "")
return minio.ErrorRespToObjectError(err, bucket)
}
return nil
}
// GetBucketPolicy will get policy on bucket
func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
data, err := l.Client.GetBucketPolicy(bucket)
if err != nil {
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, minio.ErrorRespToObjectError(err, bucket, "")
return nil, minio.ErrorRespToObjectError(err, bucket)
}
var policyInfo policy.BucketAccessPolicy
if err = json.Unmarshal([]byte(data), &policyInfo); err != nil {
return policyInfo, err
}
return policyInfo, nil
bucketPolicy, err := policy.ParseConfig(strings.NewReader(data), bucket)
return bucketPolicy, minio.ErrorRespToObjectError(err, bucket)
}
// DeleteBucketPolicy deletes all policies on bucket

View File

@@ -52,7 +52,7 @@ func TestS3ToObjectError(t *testing.T) {
},
{
inputErr: errResponse("NoSuchBucketPolicy"),
expectedErr: minio.PolicyNotFound{},
expectedErr: minio.BucketPolicyNotFound{},
},
{
inputErr: errResponse("NoSuchBucket"),