Enhance policy handling to support SSE and WORM (#5790)

- remove old bucket policy handling
- add new policy handling
- add new policy handling unit tests

This patch brings support to bucket policy to have more control not
limiting to anonymous.  Bucket owner controls to allow/deny any rest
API.

For example server side encryption can be controlled by allowing
PUT/GET objects with encryptions including bucket owner.

cmd/bucket-policy-handlers.go

@@ -19,206 +19,24 @@ package cmd
 import (
 	"encoding/json"
 	"io"
-	"io/ioutil"
-	"net"
 	"net/http"
-	"runtime"
-	"strings"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/wildcard"
+	"github.com/minio/minio/pkg/policy"
 )
 
-// maximum supported access policy size.
-const maxAccessPolicySize = 20 * humanize.KiByte
+const (
+	// As per AWS S3 specification, 20KiB policy JSON data is allowed.
+	maxBucketPolicySize = 20 * humanize.KiByte
 
-// Verify if a given action is valid for the url path based on the
-// existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions policy.ConditionKeyMap,
-	statements []policy.Statement) bool {
-	for _, statement := range statements {
-		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
-			if statement.Effect == "Allow" {
-				return true
-			}
-			// Do not uncomment kept here for readability.
-			// else statement.Effect == "Deny"
-			return false
-		}
-	}
-	// None match so deny.
-	return false
-}
+	// Policy configuration file.
+	bucketPolicyConfig = "policy.json"
+)
 
-// Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions policy.ConditionKeyMap,
-	statement policy.Statement) bool {
-	// Verify if action, resource and condition match in given statement.
-	return (bucketPolicyActionMatch(action, statement) &&
-		bucketPolicyResourceMatch(resource, statement) &&
-		bucketPolicyConditionMatch(conditions, statement))
-}
-
-// Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement policy.Statement) bool {
-	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
-}
-
-// Match function matches wild cards in 'pattern' for resource.
-func resourceMatch(pattern, resource string) bool {
-	if runtime.GOOS == "windows" {
-		// For windows specifically make sure we are case insensitive.
-		return wildcard.Match(strings.ToLower(pattern), strings.ToLower(resource))
-	}
-	return wildcard.Match(pattern, resource)
-}
-
-// Match function matches wild cards in 'pattern' for action.
-func actionMatch(pattern, action string) bool {
-	return wildcard.MatchSimple(pattern, action)
-}
-
-func refererMatch(pattern, referer string) bool {
-	return wildcard.MatchSimple(pattern, referer)
-}
-
-// isIPInCIDR - checks if a given a IP address is a member of the given subnet.
-func isIPInCIDR(cidr, ip string) bool {
-	// AWS S3 spec says IPs must use standard CIDR notation.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-3.
-	_, cidrNet, err := net.ParseCIDR(cidr)
-	if err != nil {
-		return false // If provided CIDR can't be parsed no IP will be in the subnet.
-	}
-	addr := net.ParseIP(ip)
-	return cidrNet.Contains(addr)
-}
-
-// Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool {
-	// the resource rule for object could contain "*" wild card.
-	// the requested object can be given access based on the already set bucket policy if
-	// the match is successful.
-	// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html.
-	return !statement.Resources.FuncMatch(resourceMatch, resource).IsEmpty()
-}
-
-// Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement policy.Statement) bool {
-	// Supports following conditions.
-	// - StringEquals
-	// - StringNotEquals
-	// - StringLike
-	// - StringNotLike
-	// - IpAddress
-	// - NotIpAddress
-	//
-	// Supported applicable condition keys for each conditions.
-	// - s3:prefix
-	// - s3:max-keys
-	// - s3:aws-Referer
-	// - s3:aws-SourceIp
-
-	// The following loop evaluates the logical AND of all the
-	// conditions in the statement. Note: we can break out of the
-	// loop if and only if a condition evaluates to false.
-	for condition, conditionKeyVal := range statement.Conditions {
-		prefixConditon := conditionKeyVal["s3:prefix"]
-		maxKeyCondition := conditionKeyVal["s3:max-keys"]
-		if condition == "StringEquals" {
-			// If there is no condition with "s3:prefix" or "s3:max-keys" condition key
-			// then there is nothing to check condition against.
-			if !prefixConditon.IsEmpty() && !prefixConditon.Equals(conditions["prefix"]) {
-				return false
-			}
-			if !maxKeyCondition.IsEmpty() && !maxKeyCondition.Equals(conditions["max-keys"]) {
-				return false
-			}
-		} else if condition == "StringNotEquals" {
-			// If there is no condition with "s3:prefix" or "s3:max-keys" condition key
-			// then there is nothing to check condition against.
-			if !prefixConditon.IsEmpty() && prefixConditon.Equals(conditions["prefix"]) {
-				return false
-			}
-			if !maxKeyCondition.IsEmpty() && maxKeyCondition.Equals(conditions["max-keys"]) {
-				return false
-			}
-		} else if condition == "StringLike" {
-			awsReferers := conditionKeyVal["aws:Referer"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsReferers.IsEmpty() {
-				continue
-			}
-			// wildcard match of referer in statement was not empty.
-			// StringLike has a match, i.e, condition evaluates to true.
-			refererFound := false
-			for referer := range conditions["referer"] {
-				if !awsReferers.FuncMatch(refererMatch, referer).IsEmpty() {
-					refererFound = true
-					break
-				}
-			}
-			// No matching referer found, so the condition is false.
-			if !refererFound {
-				return false
-			}
-		} else if condition == "StringNotLike" {
-			awsReferers := conditionKeyVal["aws:Referer"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsReferers.IsEmpty() {
-				continue
-			}
-			// wildcard match of referer in statement was not empty.
-			// StringNotLike has a match, i.e, condition evaluates to false.
-			for referer := range conditions["referer"] {
-				if !awsReferers.FuncMatch(refererMatch, referer).IsEmpty() {
-					return false
-				}
-			}
-		} else if condition == "IpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find a valid ip.
-			ipFound := false
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					ipFound = true
-					break
-				}
-			}
-			if !ipFound {
-				return false
-			}
-		} else if condition == "NotIpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find if nothing matches.
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					return false
-				}
-			}
-		}
-	}
-
-	return true
-}
-
-// PutBucketPolicyHandler - PUT Bucket policy
-// -----------------
-// This implementation of the PUT operation uses the policy
-// subresource to add to or replace a policy on a bucket
+// PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
+// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "PutBucketPolicy")
 
@@ -228,65 +46,46 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// If Content-Length is unknown or zero, deny the
-	// request. PutBucketPolicy always needs a Content-Length.
-	if r.ContentLength == -1 || r.ContentLength == 0 {
+	// Error out if Content-Length is missing.
+	// PutBucketPolicy always needs Content-Length.
+	if r.ContentLength <= 0 {
 		writeErrorResponse(w, ErrMissingContentLength, r.URL)
 		return
 	}
-	// If Content-Length is greater than maximum allowed policy size.
-	if r.ContentLength > maxAccessPolicySize {
+
+	// Error out if Content-Length is beyond allowed size.
+	if r.ContentLength > maxBucketPolicySize {
 		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
 		return
 	}
 
-	// Read access policy up to maxAccessPolicySize.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-	// bucket policies are limited to 20KB in size, using a limit reader.
-	policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
-	policyInfo := policy.BucketAccessPolicy{}
-	if err = json.Unmarshal(policyBytes, &policyInfo); err != nil {
-		writeErrorResponse(w, ErrInvalidPolicyDocument, r.URL)
-		return
-	}
-	// Parse check bucket policy.
-	if s3Error := checkBucketPolicyResources(bucket, policyInfo); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
-	if err = objAPI.SetBucketPolicy(ctx, bucket, policyInfo); err != nil {
-		switch err.(type) {
-		case NotImplemented:
-			// Return error for invalid bucket name.
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
-		default:
-			writeErrorResponse(w, ErrMalformedPolicy, r.URL)
-		}
-		return
-	}
-
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
+	globalPolicySys.Set(bucket, *bucketPolicy)
+	for addr, err := range globalNotificationSys.SetBucketPolicy(bucket, bucketPolicy) {
 		logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
 		logger.LogIf(ctx, err)
 	}
@@ -295,10 +94,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	writeSuccessNoContent(w)
 }
 
-// DeleteBucketPolicyHandler - DELETE Bucket policy
-// -----------------
-// This implementation of the DELETE operation uses the policy
-// subresource to add to remove a policy on a bucket.
+// DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "DeleteBucketPolicy")
 
@@ -308,29 +104,27 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// Delete bucket access policy, by passing an empty policy
-	// struct.
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
+	globalPolicySys.Remove(bucket)
+	for addr, err := range globalNotificationSys.RemoveBucketPolicy(bucket) {
 		logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
 		logger.LogIf(ctx, err)
 	}
@@ -339,10 +133,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	writeSuccessNoContent(w)
 }
 
-// GetBucketPolicyHandler - GET Bucket policy
-// -----------------
-// This operation uses the policy
-// subresource to return the policy of a specified bucket.
+// GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "GetBucketPolicy")
 
@@ -352,29 +143,28 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Read bucket access policy.
-	policy, err := objAPI.GetBucketPolicy(ctx, bucket)
+	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	policyBytes, err := json.Marshal(&policy)
+	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -382,5 +172,5 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Write to client.
-	w.Write(policyBytes)
+	w.Write(policyData)
 }