From 50a68a1791919eb7690ecbb76704c7690a1dcf6f Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 21 Sep 2021 09:02:15 -0700 Subject: [PATCH] allow S3 gateway to support object locked buckets (#13257) - Supports object locked buckets that require PutObject() to set content-md5 always. - Use SSE-S3 when S3 gateway is being used instead of SSE-KMS for auto-encryption. --- cmd/bucket-encryption.go | 8 +-- cmd/bucket-handlers.go | 6 ++- cmd/bucket-lifecycle.go | 6 +-- cmd/gateway/s3/gateway-s3.go | 4 ++ cmd/object-handlers.go | 21 ++++++-- .../bucket/encryption/bucket-sse-config.go | 51 +++++++++++-------- .../encryption/bucket-sse-config_test.go | 8 +-- 7 files changed, 68 insertions(+), 36 deletions(-) diff --git a/cmd/bucket-encryption.go b/cmd/bucket-encryption.go index 245b2d5f1..d1136d16d 100644 --- a/cmd/bucket-encryption.go +++ b/cmd/bucket-encryption.go @@ -21,7 +21,7 @@ import ( "errors" "io" - bucketsse "github.com/minio/minio/internal/bucket/encryption" + sse "github.com/minio/minio/internal/bucket/encryption" ) // BucketSSEConfigSys - in-memory cache of bucket encryption config @@ -33,7 +33,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys { } // Get - gets bucket encryption config for the given bucket. -func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) { +func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) { if globalIsGateway { objAPI := newObjectLayerFn() if objAPI == nil { @@ -47,8 +47,8 @@ func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, e } // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO. -func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) { - encConfig, err := bucketsse.ParseBucketSSEConfig(r) +func validateBucketSSEConfig(r io.Reader) (*sse.BucketSSEConfig, error) { + encConfig, err := sse.ParseBucketSSEConfig(r) if err != nil { return nil, err } diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index fe0b327b9..9ad854624 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -39,6 +39,7 @@ import ( "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio-go/v7/pkg/tags" + sse "github.com/minio/minio/internal/bucket/encryption" objectlock "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" "github.com/minio/minio/internal/config/dns" @@ -976,7 +977,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h // Check if bucket encryption is enabled sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) - sseConfig.Apply(r.Header, globalAutoEncryption) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway, + }) // get gateway encryption options var opts ObjectOptions diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index 46d5b6947..345fbf097 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -380,9 +380,9 @@ const ( // Encryption specifies encryption setting on restored bucket type Encryption struct { - EncryptionType sse.SSEAlgorithm `xml:"EncryptionType"` - KMSContext string `xml:"KMSContext,omitempty"` - KMSKeyID string `xml:"KMSKeyId,omitempty"` + EncryptionType sse.Algorithm `xml:"EncryptionType"` + KMSContext string `xml:"KMSContext,omitempty"` + KMSKeyID string `xml:"KMSKeyId,omitempty"` } // MetadataEntry denotes name and value. diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index 5f0946656..93198d9bb 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -481,6 +481,10 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, UserMetadata: opts.UserDefined, ServerSideEncryption: opts.ServerSideEncryption, UserTags: tagMap, + // Content-Md5 is needed for buckets with object locking, + // instead of spending an extra API call to detect this + // we can set md5sum to be calculated always. + SendContentMd5: true, } ui, err := l.Client.PutObject(ctx, bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), putOpts) if err != nil { diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index e865d3f26..7af249586 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -41,6 +41,7 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/tags" + sse "github.com/minio/minio/internal/bucket/encryption" "github.com/minio/minio/internal/bucket/lifecycle" objectlock "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" @@ -997,7 +998,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // Check if bucket encryption is enabled sseConfig, _ := globalBucketSSEConfigSys.Get(dstBucket) - sseConfig.Apply(r.Header, globalAutoEncryption) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway, + }) var srcOpts, dstOpts ObjectOptions srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject) @@ -1667,7 +1671,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req // Check if bucket encryption is enabled sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) - sseConfig.Apply(r.Header, globalAutoEncryption) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway, + }) actualSize := size if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 { @@ -1990,7 +1997,10 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h // Check if bucket encryption is enabled sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) - sseConfig.Apply(r.Header, globalAutoEncryption) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway, + }) retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectRetentionAction) holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction) @@ -2186,7 +2196,10 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r // Check if bucket encryption is enabled sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) - sseConfig.Apply(r.Header, globalAutoEncryption) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway, + }) // Validate storage class metadata if present if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" { diff --git a/internal/bucket/encryption/bucket-sse-config.go b/internal/bucket/encryption/bucket-sse-config.go index 00806ea1a..d23112eb9 100644 --- a/internal/bucket/encryption/bucket-sse-config.go +++ b/internal/bucket/encryption/bucket-sse-config.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package cmd +package sse import ( "encoding/xml" @@ -29,16 +29,16 @@ import ( const ( // AES256 is used with SSE-S3 - AES256 SSEAlgorithm = "AES256" + AES256 Algorithm = "AES256" // AWSKms is used with SSE-KMS - AWSKms SSEAlgorithm = "aws:kms" + AWSKms Algorithm = "aws:kms" ) -// SSEAlgorithm - represents valid SSE algorithms supported; currently only AES256 is supported -type SSEAlgorithm string +// Algorithm - represents valid SSE algorithms supported; currently only AES256 is supported +type Algorithm string // UnmarshalXML - Unmarshals XML tag to valid SSE algorithm -func (alg *SSEAlgorithm) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { +func (alg *Algorithm) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { var s string if err := d.DecodeElement(&s, &start); err != nil { return err @@ -57,18 +57,18 @@ func (alg *SSEAlgorithm) UnmarshalXML(d *xml.Decoder, start xml.StartElement) er } // MarshalXML - Marshals given SSE algorithm to valid XML -func (alg *SSEAlgorithm) MarshalXML(e *xml.Encoder, start xml.StartElement) error { +func (alg *Algorithm) MarshalXML(e *xml.Encoder, start xml.StartElement) error { return e.EncodeElement(string(*alg), start) } // EncryptionAction - for ApplyServerSideEncryptionByDefault XML tag type EncryptionAction struct { - Algorithm SSEAlgorithm `xml:"SSEAlgorithm,omitempty"` - MasterKeyID string `xml:"KMSMasterKeyID,omitempty"` + Algorithm Algorithm `xml:"SSEAlgorithm,omitempty"` + MasterKeyID string `xml:"KMSMasterKeyID,omitempty"` } -// SSERule - for ServerSideEncryptionConfiguration XML tag -type SSERule struct { +// Rule - for ServerSideEncryptionConfiguration XML tag +type Rule struct { DefaultEncryptionAction EncryptionAction `xml:"ApplyServerSideEncryptionByDefault"` } @@ -76,9 +76,9 @@ const xmlNS = "http://s3.amazonaws.com/doc/2006-03-01/" // BucketSSEConfig - represents default bucket encryption configuration type BucketSSEConfig struct { - XMLNS string `xml:"xmlns,attr,omitempty"` - XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` - Rules []SSERule `xml:"Rule"` + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` + Rules []Rule `xml:"Rule"` } // ParseBucketSSEConfig - Decodes given XML to a valid default bucket encryption config @@ -114,24 +114,35 @@ func ParseBucketSSEConfig(r io.Reader) (*BucketSSEConfig, error) { return &config, nil } +// ApplyOptions ask for specific features to be enabled, +// when bucketSSEConfig is empty. +type ApplyOptions struct { + AutoEncrypt bool + Passthrough bool // Set to 'true' for S3 gateway mode. +} + // Apply applies the SSE bucket configuration on the given HTTP headers and // sets the specified SSE headers. // // Apply does not overwrite any existing SSE headers. Further, it will // set minimal SSE-KMS headers if autoEncrypt is true and the BucketSSEConfig // is nil. -func (b *BucketSSEConfig) Apply(headers http.Header, autoEncrypt bool) { +func (b *BucketSSEConfig) Apply(headers http.Header, opts ApplyOptions) { if _, ok := crypto.IsRequested(headers); ok { return } if b == nil { - if autoEncrypt { - headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + if opts.AutoEncrypt { + if !opts.Passthrough { + headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + } else { + headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + } } return } - switch b.Algorithm() { + switch b.Algo() { case xhttp.AmzEncryptionAES: headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) case xhttp.AmzEncryptionKMS: @@ -140,8 +151,8 @@ func (b *BucketSSEConfig) Apply(headers http.Header, autoEncrypt bool) { } } -// Algorithm returns the SSE algorithm specified by the SSE configuration. -func (b *BucketSSEConfig) Algorithm() SSEAlgorithm { +// Algo returns the SSE algorithm specified by the SSE configuration. +func (b *BucketSSEConfig) Algo() Algorithm { for _, rule := range b.Rules { return rule.DefaultEncryptionAction.Algorithm } diff --git a/internal/bucket/encryption/bucket-sse-config_test.go b/internal/bucket/encryption/bucket-sse-config_test.go index 77e7352a2..d1f6c6b42 100644 --- a/internal/bucket/encryption/bucket-sse-config_test.go +++ b/internal/bucket/encryption/bucket-sse-config_test.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package cmd +package sse import ( "bytes" @@ -30,7 +30,7 @@ func TestParseBucketSSEConfig(t *testing.T) { XMLName: xml.Name{ Local: "ServerSideEncryptionConfiguration", }, - Rules: []SSERule{ + Rules: []Rule{ { DefaultEncryptionAction: EncryptionAction{ Algorithm: AES256, @@ -44,7 +44,7 @@ func TestParseBucketSSEConfig(t *testing.T) { XMLName: xml.Name{ Local: "ServerSideEncryptionConfiguration", }, - Rules: []SSERule{ + Rules: []Rule{ { DefaultEncryptionAction: EncryptionAction{ Algorithm: AES256, @@ -58,7 +58,7 @@ func TestParseBucketSSEConfig(t *testing.T) { XMLName: xml.Name{ Local: "ServerSideEncryptionConfiguration", }, - Rules: []SSERule{ + Rules: []Rule{ { DefaultEncryptionAction: EncryptionAction{ Algorithm: AWSKms,