MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: SSE-KMS use uuid instead of read all data to md5. (#17958)

Browse Source
This commit is contained in:
jiuker
2023-09-19 01:00:54 +08:00
committed by GitHub
parent a00db4267c
commit 9947c01c8e
23 changed files with 133 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/hash/reader.go
View File

@@ -19,6 +19,7 @@ package hash
import (
"bytes"
"context"
"encoding/base64"
"encoding/hex"
"errors"
@@ -71,13 +72,14 @@ type Options struct {
Size int64
ActualSize int64
DisableMD5 bool
ForceMD5 []byte
}
// NewReaderWithOpts is like NewReader but takes `Options` as argument, allowing
// callers to indicate if they want to disable md5sum checksum.
func NewReaderWithOpts(src io.Reader, opts Options) (*Reader, error) {
func NewReaderWithOpts(ctx context.Context, src io.Reader, opts Options) (*Reader, error) {
// return hard limited reader
return newReader(src, opts.Size, opts.MD5Hex, opts.SHA256Hex, opts.ActualSize, opts.DisableMD5)
return newReader(ctx, src, opts.Size, opts.MD5Hex, opts.SHA256Hex, opts.ActualSize, opts.DisableMD5, opts.ForceMD5)
}
// NewReader returns a new Reader that wraps src and computes
@@ -95,11 +97,11 @@ func NewReaderWithOpts(src io.Reader, opts Options) (*Reader, error) {
// checksums multiple times.
// NewReader enforces S3 compatibility strictly by ensuring caller
// does not send more content than specified size.
func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) {
return newReader(src, size, md5Hex, sha256Hex, actualSize, false)
func NewReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) {
return newReader(ctx, src, size, md5Hex, sha256Hex, actualSize, false, nil)
}
func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64, disableMD5 bool) (*Reader, error) {
func newReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64, disableMD5 bool, forceMD5 []byte) (*Reader, error) {
MD5, err := hex.DecodeString(md5Hex)
if err != nil {
return nil, BadDigest{ // TODO(aead): Return an error that indicates that an invalid ETag has been specified
@@ -153,7 +155,7 @@ func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
r := ioutil.HardLimitReader(src, size)
if !disableMD5 {
if _, ok := src.(etag.Tagger); !ok {
src = etag.NewReader(r, MD5)
src = etag.NewReader(ctx, r, MD5, forceMD5)
} else {
src = etag.Wrap(r, src)
}
@@ -162,7 +164,7 @@ func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
}
} else if _, ok := src.(etag.Tagger); !ok {
if !disableMD5 {
src = etag.NewReader(src, MD5)
src = etag.NewReader(ctx, src, MD5, forceMD5)
}
}
var h hash.Hash

9
internal/hash/reader_test.go
View File

@@ -19,6 +19,7 @@ package hash
import (
"bytes"
"context"
"encoding/base64"
"encoding/hex"
"fmt"
@@ -30,7 +31,7 @@ import (
// Tests functions like Size(), MD5*(), SHA256*()
func TestHashReaderHelperMethods(t *testing.T) {
r, err := NewReader(bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4)
r, err := NewReader(context.Background(), bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4)
if err != nil {
t.Fatal(err)
}
@@ -194,7 +195,7 @@ func TestHashReaderVerification(t *testing.T) {
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
r, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
r, err := NewReader(context.Background(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
if err != nil {
t.Fatalf("Test %q: Initializing reader failed %s", testCase.desc, err)
}
@@ -213,7 +214,7 @@ func TestHashReaderVerification(t *testing.T) {
}
func mustReader(t *testing.T, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) *Reader {
r, err := NewReader(src, size, md5Hex, sha256Hex, actualSize)
r, err := NewReader(context.Background(), src, size, md5Hex, sha256Hex, actualSize)
if err != nil {
t.Fatal(err)
}
@@ -303,7 +304,7 @@ func TestHashReaderInvalidArguments(t *testing.T) {
for i, testCase := range testCases {
t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
_, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
_, err := NewReader(context.Background(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
if err != nil && testCase.success {
t.Errorf("Test %q: Expected success, but got error %s instead", testCase.desc, err)
}