allow S3 gateway to support object locked buckets (#13257)

- Supports object locked buckets that require
  PutObject() to set content-md5 always.
- Use SSE-S3 when S3 gateway is being used instead
  of SSE-KMS for auto-encryption.

cmd/bucket-encryption.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"io"
 
-	bucketsse "github.com/minio/minio/internal/bucket/encryption"
+	sse "github.com/minio/minio/internal/bucket/encryption"
 )
 
 // BucketSSEConfigSys - in-memory cache of bucket encryption config
@@ -33,7 +33,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 }
 
 // Get - gets bucket encryption config for the given bucket.
-func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
+func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerFn()
 		if objAPI == nil {
@@ -47,8 +47,8 @@ func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, e
 }
 
 // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.
-func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) {
+func validateBucketSSEConfig(r io.Reader) (*sse.BucketSSEConfig, error) {
-	encConfig, err := bucketsse.ParseBucketSSEConfig(r)
+	encConfig, err := sse.ParseBucketSSEConfig(r)
 	if err != nil {
 		return nil, err
 	}

cmd/bucket-handlers.go

@@ -39,6 +39,7 @@ import (
 
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	sse "github.com/minio/minio/internal/bucket/encryption"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/config/dns"
@@ -976,7 +977,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
-	sseConfig.Apply(r.Header, globalAutoEncryption)
+	sseConfig.Apply(r.Header, sse.ApplyOptions{
+		AutoEncrypt: globalAutoEncryption,
+		Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
+	})
 
 	// get gateway encryption options
 	var opts ObjectOptions

cmd/bucket-lifecycle.go

@@ -380,7 +380,7 @@ const (
 
 // Encryption specifies encryption setting on restored bucket
 type Encryption struct {
-	EncryptionType sse.SSEAlgorithm `xml:"EncryptionType"`
+	EncryptionType sse.Algorithm `xml:"EncryptionType"`
 	KMSContext     string        `xml:"KMSContext,omitempty"`
 	KMSKeyID       string        `xml:"KMSKeyId,omitempty"`
 }

cmd/gateway/s3/gateway-s3.go

@@ -481,6 +481,10 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string,
 		UserMetadata:         opts.UserDefined,
 		ServerSideEncryption: opts.ServerSideEncryption,
 		UserTags:             tagMap,
+		// Content-Md5 is needed for buckets with object locking,
+		// instead of spending an extra API call to detect this
+		// we can set md5sum to be calculated always.
+		SendContentMd5: true,
 	}
 	ui, err := l.Client.PutObject(ctx, bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), putOpts)
 	if err != nil {

cmd/object-handlers.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	sse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -997,7 +998,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(dstBucket)
-	sseConfig.Apply(r.Header, globalAutoEncryption)
+	sseConfig.Apply(r.Header, sse.ApplyOptions{
+		AutoEncrypt: globalAutoEncryption,
+		Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
+	})
 
 	var srcOpts, dstOpts ObjectOptions
 	srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject)
@@ -1667,7 +1671,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
-	sseConfig.Apply(r.Header, globalAutoEncryption)
+	sseConfig.Apply(r.Header, sse.ApplyOptions{
+		AutoEncrypt: globalAutoEncryption,
+		Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
+	})
 
 	actualSize := size
 	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
@@ -1990,7 +1997,10 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 
 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
-	sseConfig.Apply(r.Header, globalAutoEncryption)
+	sseConfig.Apply(r.Header, sse.ApplyOptions{
+		AutoEncrypt: globalAutoEncryption,
+		Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
+	})
 
 	retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction)
@@ -2186,7 +2196,10 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 
 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
-	sseConfig.Apply(r.Header, globalAutoEncryption)
+	sseConfig.Apply(r.Header, sse.ApplyOptions{
+		AutoEncrypt: globalAutoEncryption,
+		Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
+	})
 
 	// Validate storage class metadata if present
 	if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {

internal/bucket/encryption/bucket-sse-config.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package cmd
+package sse
 
 import (
 	"encoding/xml"
@@ -29,16 +29,16 @@ import (
 
 const (
 	// AES256 is used with SSE-S3
-	AES256 SSEAlgorithm = "AES256"
+	AES256 Algorithm = "AES256"
 	// AWSKms is used with SSE-KMS
-	AWSKms SSEAlgorithm = "aws:kms"
+	AWSKms Algorithm = "aws:kms"
 )
 
-// SSEAlgorithm - represents valid SSE algorithms supported; currently only AES256 is supported
+// Algorithm - represents valid SSE algorithms supported; currently only AES256 is supported
-type SSEAlgorithm string
+type Algorithm string
 
 // UnmarshalXML - Unmarshals XML tag to valid SSE algorithm
-func (alg *SSEAlgorithm) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+func (alg *Algorithm) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var s string
 	if err := d.DecodeElement(&s, &start); err != nil {
 		return err
@@ -57,18 +57,18 @@ func (alg *SSEAlgorithm) UnmarshalXML(d *xml.Decoder, start xml.StartElement) er
 }
 
 // MarshalXML - Marshals given SSE algorithm to valid XML
-func (alg *SSEAlgorithm) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+func (alg *Algorithm) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	return e.EncodeElement(string(*alg), start)
 }
 
 // EncryptionAction - for ApplyServerSideEncryptionByDefault XML tag
 type EncryptionAction struct {
-	Algorithm   SSEAlgorithm `xml:"SSEAlgorithm,omitempty"`
+	Algorithm   Algorithm `xml:"SSEAlgorithm,omitempty"`
 	MasterKeyID string    `xml:"KMSMasterKeyID,omitempty"`
 }
 
-// SSERule - for ServerSideEncryptionConfiguration XML tag
+// Rule - for ServerSideEncryptionConfiguration XML tag
-type SSERule struct {
+type Rule struct {
 	DefaultEncryptionAction EncryptionAction `xml:"ApplyServerSideEncryptionByDefault"`
 }
 
@@ -78,7 +78,7 @@ const xmlNS = "http://s3.amazonaws.com/doc/2006-03-01/"
 type BucketSSEConfig struct {
 	XMLNS   string   `xml:"xmlns,attr,omitempty"`
 	XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"`
-	Rules   []SSERule `xml:"Rule"`
+	Rules   []Rule   `xml:"Rule"`
 }
 
 // ParseBucketSSEConfig - Decodes given XML to a valid default bucket encryption config
@@ -114,24 +114,35 @@ func ParseBucketSSEConfig(r io.Reader) (*BucketSSEConfig, error) {
 	return &config, nil
 }
 
+// ApplyOptions ask for specific features to be enabled,
+// when bucketSSEConfig is empty.
+type ApplyOptions struct {
+	AutoEncrypt bool
+	Passthrough bool // Set to 'true' for S3 gateway mode.
+}
+
 // Apply applies the SSE bucket configuration on the given HTTP headers and
 // sets the specified SSE headers.
 //
 // Apply does not overwrite any existing SSE headers. Further, it will
 // set minimal SSE-KMS headers if autoEncrypt is true and the BucketSSEConfig
 // is nil.
-func (b *BucketSSEConfig) Apply(headers http.Header, autoEncrypt bool) {
+func (b *BucketSSEConfig) Apply(headers http.Header, opts ApplyOptions) {
 	if _, ok := crypto.IsRequested(headers); ok {
 		return
 	}
 	if b == nil {
-		if autoEncrypt {
+		if opts.AutoEncrypt {
+			if !opts.Passthrough {
 				headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
+			} else {
+				headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+			}
 		}
 		return
 	}
 
-	switch b.Algorithm() {
+	switch b.Algo() {
 	case xhttp.AmzEncryptionAES:
 		headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 	case xhttp.AmzEncryptionKMS:
@@ -140,8 +151,8 @@ func (b *BucketSSEConfig) Apply(headers http.Header, autoEncrypt bool) {
 	}
 }
 
-// Algorithm returns the SSE algorithm specified by the SSE configuration.
+// Algo returns the SSE algorithm specified by the SSE configuration.
-func (b *BucketSSEConfig) Algorithm() SSEAlgorithm {
+func (b *BucketSSEConfig) Algo() Algorithm {
 	for _, rule := range b.Rules {
 		return rule.DefaultEncryptionAction.Algorithm
 	}

internal/bucket/encryption/bucket-sse-config_test.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package cmd
+package sse
 
 import (
 	"bytes"
@@ -30,7 +30,7 @@ func TestParseBucketSSEConfig(t *testing.T) {
 		XMLName: xml.Name{
 			Local: "ServerSideEncryptionConfiguration",
 		},
-		Rules: []SSERule{
+		Rules: []Rule{
 			{
 				DefaultEncryptionAction: EncryptionAction{
 					Algorithm: AES256,
@@ -44,7 +44,7 @@ func TestParseBucketSSEConfig(t *testing.T) {
 		XMLName: xml.Name{
 			Local: "ServerSideEncryptionConfiguration",
 		},
-		Rules: []SSERule{
+		Rules: []Rule{
 			{
 				DefaultEncryptionAction: EncryptionAction{
 					Algorithm: AES256,
@@ -58,7 +58,7 @@ func TestParseBucketSSEConfig(t *testing.T) {
 		XMLName: xml.Name{
 			Local: "ServerSideEncryptionConfiguration",
 		},
-		Rules: []SSERule{
+		Rules: []Rule{
 			{
 				DefaultEncryptionAction: EncryptionAction{
 					Algorithm:   AWSKms,