diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index ae612cdbd..bc8e6fcba 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -865,7 +865,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } - if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok { + if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) return } @@ -1041,7 +1041,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } if objectAPI.IsEncryptionSupported() { - if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests + if crypto.Requested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests if crypto.SSECopy.IsRequested(r.Header) { writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL) return diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 0432ee8be..a84d76d4e 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -976,14 +976,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st switch size := data.Size(); { case size == 0: buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF - case size == -1: - if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize { - buffer = make([]byte, data.ActualSize()+256, data.ActualSize()*2+512) - } else { - buffer = er.bp.Get() - defer er.bp.Put(buffer) - } - case size >= fi.Erasure.BlockSize: + case size >= fi.Erasure.BlockSize || size == -1: buffer = er.bp.Get() defer er.bp.Put(buffer) case size < fi.Erasure.BlockSize: diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index fabdda2f3..491f46741 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -74,6 +74,10 @@ const ( compReadAheadBuffers = 5 // Size of each buffer. compReadAheadBufSize = 1 << 20 + // Pad Encrypted+Compressed files to a multiple of this. + compPadEncrypted = 256 + // Disable compressed file indices below this size + compMinIndexSize = 8 << 20 ) // isMinioBucket returns true if given bucket is a MinIO internal @@ -436,8 +440,7 @@ func isCompressible(header http.Header, object string) bool { cfg := globalCompressConfig globalCompressConfigMu.Unlock() - _, ok := crypto.IsRequested(header) - if !cfg.Enabled || (ok && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) { + if !cfg.Enabled || (crypto.Requested(header) && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) { return false } return true @@ -985,10 +988,17 @@ func init() { // input 'on' is always recommended such that this function works // properly, because we do not wish to create an object even if // client closed the stream prematurely. -func newS2CompressReader(r io.Reader, on int64) (rc io.ReadCloser, idx func() []byte) { +func newS2CompressReader(r io.Reader, on int64, encrypted bool) (rc io.ReadCloser, idx func() []byte) { pr, pw := io.Pipe() // Copy input to compressor - comp := s2.NewWriter(pw, compressOpts...) + opts := compressOpts + if encrypted { + // The values used for padding are not a security concern, + // but we choose pseudo-random numbers instead of just zeros. + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + opts = append([]s2.WriterOption{s2.WriterPadding(compPadEncrypted), s2.WriterPaddingSrc(rng)}, compressOpts...) + } + comp := s2.NewWriter(pw, opts...) indexCh := make(chan []byte, 1) go func() { defer close(indexCh) @@ -1006,8 +1016,8 @@ func newS2CompressReader(r io.Reader, on int64) (rc io.ReadCloser, idx func() [] return } // Close the stream. - // If more than 8MB was written, generate index. - if cn > 8<<20 { + // If more than compMinIndexSize was written, generate index. + if cn > compMinIndexSize { idx, err := comp.CloseIndex() idx = s2.RemoveIndexHeaders(idx) indexCh <- idx @@ -1048,7 +1058,7 @@ func compressSelfTest() { } } const skip = 2<<20 + 511 - r, _ := newS2CompressReader(bytes.NewBuffer(data), int64(len(data))) + r, _ := newS2CompressReader(bytes.NewBuffer(data), int64(len(data)), true) b, err := io.ReadAll(r) failOnErr(err) failOnErr(r.Close()) diff --git a/cmd/object-api-utils_test.go b/cmd/object-api-utils_test.go index 3429727a9..349a00802 100644 --- a/cmd/object-api-utils_test.go +++ b/cmd/object-api-utils_test.go @@ -624,7 +624,7 @@ func TestS2CompressReader(t *testing.T) { t.Run(tt.name, func(t *testing.T) { buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case - r, idxCB := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data))) + r, idxCB := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data)), false) defer r.Close() var rdrBuf bytes.Buffer diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 44b8ff0a6..01b4d6af8 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -119,7 +119,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r return } - if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() { + if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) return } @@ -330,7 +330,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) return } - if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok { + if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) return } @@ -611,7 +611,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest)) return } - if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok { + if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) return } @@ -951,7 +951,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return } - if _, ok := crypto.IsRequested(r.Header); ok { + if crypto.Requested(r.Header) { if globalIsGateway { if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) @@ -1151,7 +1151,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } // Check if either the source is encrypted or the destination will be encrypted. - _, objectEncryption := crypto.IsRequested(r.Header) + objectEncryption := crypto.Requested(r.Header) objectEncryption = objectEncryption || crypto.IsSourceEncrypted(srcInfo.UserDefined) var compressMetadata map[string]string @@ -1168,7 +1168,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10) reader = etag.NewReader(reader, nil) - s2c, cb := newS2CompressReader(reader, actualSize) + wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption) dstOpts.IndexCB = cb defer s2c.Close() reader = etag.Wrap(s2c, reader) @@ -1573,7 +1574,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req return } - if _, ok := crypto.IsRequested(r.Header); ok { + if crypto.Requested(r.Header) { if globalIsGateway { if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) @@ -1737,7 +1738,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req // Set compression metrics. var s2c io.ReadCloser - s2c, idxCb = newS2CompressReader(actualReader, actualSize) + wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header) + s2c, idxCb = newS2CompressReader(actualReader, actualSize, wantEncryption) defer s2c.Close() reader = etag.Wrap(s2c, actualReader) @@ -1796,7 +1798,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req } var objectEncryptionKey crypto.ObjectKey if objectAPI.IsEncryptionSupported() { - if _, ok := crypto.IsRequested(r.Header); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests + if crypto.Requested(r.Header) && !HasSuffix(object, SlashSeparator) { // handle SSE requests if crypto.SSECopy.IsRequested(r.Header) { writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL) return @@ -1929,7 +1931,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h return } - if _, ok := crypto.IsRequested(r.Header); ok { + if crypto.Requested(r.Header) { if globalIsGateway { if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) @@ -2088,7 +2090,8 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h } // Set compression metrics. - s2c, cb := newS2CompressReader(actualReader, actualSize) + wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(actualReader, actualSize, wantEncryption) defer s2c.Close() idxCb = cb reader = etag.Wrap(s2c, actualReader) @@ -2144,7 +2147,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h var objectEncryptionKey crypto.ObjectKey if objectAPI.IsEncryptionSupported() { - if _, ok := crypto.IsRequested(r.Header); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests + if crypto.Requested(r.Header) && !HasSuffix(object, SlashSeparator) { // handle SSE requests if crypto.SSECopy.IsRequested(r.Header) { return errInvalidEncryptionParameters } @@ -2234,7 +2237,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r return } - if _, ok := crypto.IsRequested(r.Header); ok { + if crypto.Requested(r.Header) { if globalIsGateway { if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) @@ -2279,7 +2282,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r encMetadata := map[string]string{} if objectAPI.IsEncryptionSupported() { - if _, ok := crypto.IsRequested(r.Header); ok { + if crypto.Requested(r.Header) { if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -2377,7 +2380,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt return } - if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok { + if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) return } @@ -2594,7 +2597,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt // Compress only if the compression is enabled during initial multipart. var idxCb func() []byte if isCompressed { - s2c, cb := newS2CompressReader(reader, actualPartSize) + wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(reader, actualPartSize, wantEncryption) idxCb = cb defer s2c.Close() reader = etag.Wrap(s2c, reader) @@ -2709,7 +2713,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http return } - if _, ok := crypto.IsRequested(r.Header); ok { + if crypto.Requested(r.Header) { if globalIsGateway { if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) @@ -2857,7 +2861,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http } // Set compression metrics. - s2c, cb := newS2CompressReader(actualReader, actualSize) + wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(actualReader, actualSize, wantEncryption) idxCb = cb defer s2c.Close() reader = etag.Wrap(s2c, actualReader) diff --git a/cmd/s3-zip-handlers.go b/cmd/s3-zip-handlers.go index 8b5dbfc9c..0d948cc36 100644 --- a/cmd/s3-zip-handlers.go +++ b/cmd/s3-zip-handlers.go @@ -66,7 +66,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context, writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) return } - if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok { + if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) return } @@ -357,7 +357,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context, writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest)) return } - if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok { + if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) return } diff --git a/internal/bucket/encryption/bucket-sse-config.go b/internal/bucket/encryption/bucket-sse-config.go index d23112eb9..13e6114f6 100644 --- a/internal/bucket/encryption/bucket-sse-config.go +++ b/internal/bucket/encryption/bucket-sse-config.go @@ -128,7 +128,7 @@ type ApplyOptions struct { // set minimal SSE-KMS headers if autoEncrypt is true and the BucketSSEConfig // is nil. func (b *BucketSSEConfig) Apply(headers http.Header, opts ApplyOptions) { - if _, ok := crypto.IsRequested(headers); ok { + if crypto.Requested(headers) { return } if b == nil { diff --git a/internal/crypto/header_test.go b/internal/crypto/header_test.go index 02ab3cd8a..424d889fe 100644 --- a/internal/crypto/header_test.go +++ b/internal/crypto/header_test.go @@ -29,6 +29,10 @@ import ( func TestIsRequested(t *testing.T) { for i, test := range kmsIsRequestedTests { _, got := IsRequested(test.Header) + if Requested(test.Header) != got { + // Test if result matches. + t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got) + } got = got && S3KMS.IsRequested(test.Header) if got != test.Expected { t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got) @@ -36,6 +40,10 @@ func TestIsRequested(t *testing.T) { } for i, test := range s3IsRequestedTests { _, got := IsRequested(test.Header) + if Requested(test.Header) != got { + // Test if result matches. + t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got) + } got = got && S3.IsRequested(test.Header) if got != test.Expected { t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got) @@ -43,6 +51,10 @@ func TestIsRequested(t *testing.T) { } for i, test := range ssecIsRequestedTests { _, got := IsRequested(test.Header) + if Requested(test.Header) != got { + // Test if result matches. + t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got) + } got = got && SSEC.IsRequested(test.Header) if got != test.Expected { t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got) diff --git a/internal/crypto/sse.go b/internal/crypto/sse.go index 35c2ef016..60c0c18a5 100644 --- a/internal/crypto/sse.go +++ b/internal/crypto/sse.go @@ -71,6 +71,11 @@ func IsRequested(h http.Header) (Type, bool) { } } +// Requested returns whether any type of encryption is requested. +func Requested(h http.Header) bool { + return S3.IsRequested(h) || S3KMS.IsRequested(h) || SSEC.IsRequested(h) +} + // UnsealObjectKey extracts and decrypts the sealed object key // from the metadata using the SSE-Copy client key of the HTTP headers // and returns the decrypted object key.