feat: SSE-KMS use uuid instead of read all data to md5. (#17958)

internal/etag/etag.go

@@ -119,6 +119,7 @@ import (
 
 	"github.com/minio/minio/internal/fips"
 	"github.com/minio/minio/internal/hash/sha256"
+	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/sio"
 )
 
@@ -263,6 +264,12 @@ func FromContentMD5(h http.Header) (ETag, error) {
 	return ETag(b), nil
 }
 
+// ContentMD5Requested - for http.request.header is not request Content-Md5
+func ContentMD5Requested(h http.Header) bool {
+	_, ok := h[xhttp.ContentMD5]
+	return ok
+}
+
 // Multipart computes an S3 multipart ETag given a list of
 // S3 singlepart ETags. It returns nil if the list of
 // ETags is empty.

internal/etag/etag_test.go

@@ -18,6 +18,7 @@
 package etag
 
 import (
+	"context"
 	"io"
 	"net/http"
 	"strings"
@@ -137,7 +138,7 @@ var readerTests = []struct { // Reference values computed by: echo <content> | m
 
 func TestReader(t *testing.T) {
 	for i, test := range readerTests {
-		reader := NewReader(strings.NewReader(test.Content), test.ETag)
+		reader := NewReader(context.Background(), strings.NewReader(test.Content), test.ETag, nil)
 		if _, err := io.Copy(io.Discard, reader); err != nil {
 			t.Fatalf("Test %d: read failed: %v", i, err)
 		}

internal/etag/reader.go

@@ -18,6 +18,7 @@
 package etag
 
 import (
+	"context"
 	"crypto/md5"
 	"fmt"
 	"hash"
@@ -102,12 +103,19 @@ type Reader struct {
 // If the provided etag is not nil the returned
 // Reader compares the etag with the computed
 // MD5 sum once the r returns io.EOF.
-func NewReader(r io.Reader, etag ETag) *Reader {
+func NewReader(ctx context.Context, r io.Reader, etag ETag, forceMD5 []byte) *Reader {
 	if er, ok := r.(*Reader); ok {
 		if er.readN == 0 && Equal(etag, er.checksum) {
 			return er
 		}
 	}
+	if len(forceMD5) != 0 {
+		return &Reader{
+			src:      r,
+			md5:      NewUUIDHash(forceMD5),
+			checksum: etag,
+		}
+	}
 	return &Reader{
 		src:      r,
 		md5:      md5.New(),
@@ -153,3 +161,40 @@ type VerifyError struct {
 func (v VerifyError) Error() string {
 	return fmt.Sprintf("etag: expected ETag %q does not match computed ETag %q", v.Expected, v.Computed)
 }
+
+// UUIDHash - use uuid to make md5sum
+type UUIDHash struct {
+	uuid []byte
+}
+
+// Write -  implement hash.Hash Write
+func (u UUIDHash) Write(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
+// Sum -  implement md5.Sum
+func (u UUIDHash) Sum(b []byte) []byte {
+	return u.uuid
+}
+
+// Reset -  implement hash.Hash Reset
+func (u UUIDHash) Reset() {
+	return
+}
+
+// Size -  implement hash.Hash Size
+func (u UUIDHash) Size() int {
+	return len(u.uuid)
+}
+
+// BlockSize -  implement hash.Hash BlockSize
+func (u UUIDHash) BlockSize() int {
+	return md5.BlockSize
+}
+
+var _ hash.Hash = &UUIDHash{}
+
+// NewUUIDHash - new UUIDHash
+func NewUUIDHash(uuid []byte) *UUIDHash {
+	return &UUIDHash{uuid: uuid}
+}

internal/hash/reader.go

@@ -19,6 +19,7 @@ package hash
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
@@ -71,13 +72,14 @@ type Options struct {
 	Size       int64
 	ActualSize int64
 	DisableMD5 bool
+	ForceMD5   []byte
 }
 
 // NewReaderWithOpts is like NewReader but takes `Options` as argument, allowing
 // callers to indicate if they want to disable md5sum checksum.
-func NewReaderWithOpts(src io.Reader, opts Options) (*Reader, error) {
+func NewReaderWithOpts(ctx context.Context, src io.Reader, opts Options) (*Reader, error) {
 	// return hard limited reader
-	return newReader(src, opts.Size, opts.MD5Hex, opts.SHA256Hex, opts.ActualSize, opts.DisableMD5)
+	return newReader(ctx, src, opts.Size, opts.MD5Hex, opts.SHA256Hex, opts.ActualSize, opts.DisableMD5, opts.ForceMD5)
 }
 
 // NewReader returns a new Reader that wraps src and computes
@@ -95,11 +97,11 @@ func NewReaderWithOpts(src io.Reader, opts Options) (*Reader, error) {
 // checksums multiple times.
 // NewReader enforces S3 compatibility strictly by ensuring caller
 // does not send more content than specified size.
-func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) {
-	return newReader(src, size, md5Hex, sha256Hex, actualSize, false)
+func NewReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) {
+	return newReader(ctx, src, size, md5Hex, sha256Hex, actualSize, false, nil)
 }
 
-func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64, disableMD5 bool) (*Reader, error) {
+func newReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64, disableMD5 bool, forceMD5 []byte) (*Reader, error) {
 	MD5, err := hex.DecodeString(md5Hex)
 	if err != nil {
 		return nil, BadDigest{ // TODO(aead): Return an error that indicates that an invalid ETag has been specified
@@ -153,7 +155,7 @@ func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
 		r := ioutil.HardLimitReader(src, size)
 		if !disableMD5 {
 			if _, ok := src.(etag.Tagger); !ok {
-				src = etag.NewReader(r, MD5)
+				src = etag.NewReader(ctx, r, MD5, forceMD5)
 			} else {
 				src = etag.Wrap(r, src)
 			}
@@ -162,7 +164,7 @@ func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
 		}
 	} else if _, ok := src.(etag.Tagger); !ok {
 		if !disableMD5 {
-			src = etag.NewReader(src, MD5)
+			src = etag.NewReader(ctx, src, MD5, forceMD5)
 		}
 	}
 	var h hash.Hash

internal/hash/reader_test.go

@@ -19,6 +19,7 @@ package hash
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
@@ -30,7 +31,7 @@ import (
 
 // Tests functions like Size(), MD5*(), SHA256*()
 func TestHashReaderHelperMethods(t *testing.T) {
-	r, err := NewReader(bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4)
+	r, err := NewReader(context.Background(), bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -194,7 +195,7 @@ func TestHashReaderVerification(t *testing.T) {
 	}
 	for i, testCase := range testCases {
 		t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
-			r, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
+			r, err := NewReader(context.Background(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
 			if err != nil {
 				t.Fatalf("Test %q: Initializing reader failed %s", testCase.desc, err)
 			}
@@ -213,7 +214,7 @@ func TestHashReaderVerification(t *testing.T) {
 }
 
 func mustReader(t *testing.T, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) *Reader {
-	r, err := NewReader(src, size, md5Hex, sha256Hex, actualSize)
+	r, err := NewReader(context.Background(), src, size, md5Hex, sha256Hex, actualSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -303,7 +304,7 @@ func TestHashReaderInvalidArguments(t *testing.T) {
 
 	for i, testCase := range testCases {
 		t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
-			_, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
+			_, err := NewReader(context.Background(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
 			if err != nil && testCase.success {
 				t.Errorf("Test %q: Expected success, but got error %s instead", testCase.desc, err)
 			}