MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-11 15:03:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: SSE-KMS use uuid instead of read all data to md5. (#17958)

Browse Source
This commit is contained in:
jiuker 2023-09-19 01:00:54 +08:00 committed by GitHub
parent a00db4267c
commit 9947c01c8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 133 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/auth-handler.go
View File

@ -567,7 +567,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present. // Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
// The verification happens implicit during reading. // The verification happens implicit during reading.
reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1) reader, err := hash.NewReader(ctx, r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
if err != nil { if err != nil {
return toAPIErrorCode(ctx, err) return toAPIErrorCode(ctx, err)
} }

4
cmd/batch-handlers.go
View File

@ -166,7 +166,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
} }
defer rd.Close() defer rd.Close()
hr, err := hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size) hr, err := hash.NewReader(ctx, rd, objInfo.Size, "", "", objInfo.Size)
if err != nil { if err != nil {
return err return err
} }
@ -229,7 +229,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
} }
defer rd.Close() defer rd.Close()
hr, err = hash.NewReader(io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size) hr, err = hash.NewReader(ctx, io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size)
if err != nil { if err != nil {
return err return err
} }

4
cmd/bucket-handlers.go
View File

@ -1139,7 +1139,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
return return
} }
hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize) hashReader, err := hash.NewReader(ctx, reader, fileSize, "", "", fileSize)
if err != nil { if err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@ -1254,7 +1254,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
return return
} }
// do not try to verify encrypted content/ // do not try to verify encrypted content/
hashReader, err = hash.NewReader(reader, -1, "", "", -1) hashReader, err = hash.NewReader(ctx, reader, -1, "", "", -1)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return

2
cmd/bucket-replication.go
View File

@ -1571,7 +1571,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
) )
for _, partInfo := range objInfo.Parts { for _, partInfo := range objInfo.Parts {
hr, err = hash.NewReader(io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize) hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
if err != nil { if err != nil {
return err return err
} }

2
cmd/config-common.go
View File

@ -71,7 +71,7 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
} }
func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error { func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error {
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data))) hashReader, err := hash.NewReader(ctx, bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
if err != nil { if err != nil {
return err return err
} }

2
cmd/data-usage-cache.go
View File

@ -998,7 +998,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
return err return err
} }
hr, err := hash.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", "", int64(buf.Len())) hr, err := hash.NewReader(ctx, bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", "", int64(buf.Len()))
if err != nil { if err != nil {
return err return err
} }

2
cmd/disk-cache-backend.go
View File

@ -1441,7 +1441,7 @@ func newCachePartEncryptReader(ctx context.Context, bucket, object string, partI
info := ObjectInfo{Size: size} info := ObjectInfo{Size: size}
wantSize = info.EncryptedSize() wantSize = info.EncryptedSize()
} }
hReader, err := hash.NewReader(content, wantSize, "", "", size) hReader, err := hash.NewReader(ctx, content, wantSize, "", "", size)
if err != nil { if err != nil {
return nil, err return nil, err
} }

6
cmd/disk-cache.go
View File

@ -734,7 +734,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
defer cLock.Unlock(lkctx) defer cLock.Unlock(lkctx)
// Initialize pipe to stream data to backend // Initialize pipe to stream data to backend
pipeReader, pipeWriter := io.Pipe() pipeReader, pipeWriter := io.Pipe()
hashReader, err := hash.NewReader(pipeReader, size, "", "", r.ActualSize()) hashReader, err := hash.NewReader(ctx, pipeReader, size, "", "", r.ActualSize())
if err != nil { if err != nil {
return return
} }
@ -795,7 +795,7 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
if st == CommitComplete || st.String() == "" { if st == CommitComplete || st.String() == "" {
return return
} }
hashReader, err := hash.NewReader(cReader, oi.Size, "", "", oi.Size) hashReader, err := hash.NewReader(ctx, cReader, oi.Size, "", "", oi.Size)
if err != nil { if err != nil {
return return
} }
@ -1059,7 +1059,7 @@ func (c *cacheObjects) PutObjectPart(ctx context.Context, bucket, object, upload
info = PartInfo{} info = PartInfo{}
// Initialize pipe to stream data to backend // Initialize pipe to stream data to backend
pipeReader, pipeWriter := io.Pipe() pipeReader, pipeWriter := io.Pipe()
hashReader, err := hash.NewReader(pipeReader, size, "", "", data.ActualSize()) hashReader, err := hash.NewReader(ctx, pipeReader, size, "", "", data.ActualSize())
if err != nil { if err != nil {
return return
} }

4
cmd/erasure-object.go
View File

@ -2155,7 +2155,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
} }
defer gr.Close() defer gr.Close()
hashReader, err := hash.NewReader(gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size) hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size)
if err != nil { if err != nil {
return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
} }
@ -2180,7 +2180,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
// rehydrate the parts back on disk as per the original xl.meta prior to transition // rehydrate the parts back on disk as per the original xl.meta prior to transition
for _, partInfo := range oi.Parts { for _, partInfo := range oi.Parts {
hr, err := hash.NewReader(io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size) hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
if err != nil { if err != nil {
return setRestoreHeaderFn(oi, err) return setRestoreHeaderFn(oi, err)
} }

4
cmd/erasure-server-pool-decom.go
View File

@ -608,7 +608,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{}) defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{})
parts := make([]CompletePart, len(objInfo.Parts)) parts := make([]CompletePart, len(objInfo.Parts))
for i, part := range objInfo.Parts { for i, part := range objInfo.Parts {
hr, err := hash.NewReader(io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize) hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
if err != nil { if err != nil {
return fmt.Errorf("decommissionObject: hash.NewReader() %w", err) return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
} }
@ -642,7 +642,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
return err return err
} }
hr, err := hash.NewReader(io.LimitReader(gr, objInfo.Size), objInfo.Size, "", "", actualSize) hr, err := hash.NewReader(ctx, io.LimitReader(gr, objInfo.Size), objInfo.Size, "", "", actualSize)
if err != nil { if err != nil {
return fmt.Errorf("decommissionObject: hash.NewReader() %w", err) return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
} }

4
cmd/erasure-server-pool-rebalance.go
View File

@ -736,7 +736,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
parts := make([]CompletePart, len(oi.Parts)) parts := make([]CompletePart, len(oi.Parts))
for i, part := range oi.Parts { for i, part := range oi.Parts {
hr, err := hash.NewReader(io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize) hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
if err != nil { if err != nil {
return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err) return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
} }
@ -766,7 +766,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
return err return err
} }
hr, err := hash.NewReader(gr, oi.Size, "", "", actualSize) hr, err := hash.NewReader(ctx, gr, oi.Size, "", "", actualSize)
if err != nil { if err != nil {
return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err) return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
} }

2
cmd/metacache-set.go
View File

@ -761,7 +761,7 @@ func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCache
return nil return nil
} }
o.debugln(color.Green("saveMetaCacheStream:")+" saving block", b.n, "to", o.objectPath(b.n)) o.debugln(color.Green("saveMetaCacheStream:")+" saving block", b.n, "to", o.objectPath(b.n))
r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data))) r, err := hash.NewReader(ctx, bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)))
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
custom := b.headerKV() custom := b.headerKV()
_, err = er.putMetacacheObject(ctx, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{ _, err = er.putMetacacheObject(ctx, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{

9
cmd/object-api-utils.go
View File

@ -342,6 +342,15 @@ func mustGetUUID() string {
return u.String() return u.String()
} }
// mustGetUUIDBytes - get a random UUID as 16 bytes unencoded.
func mustGetUUIDBytes() []byte {
u, err := uuid.NewRandom()
if err != nil {
logger.CriticalIf(GlobalContext, err)
}
return u[:]
}
// Create an s3 compatible MD5sum for complete multipart transaction. // Create an s3 compatible MD5sum for complete multipart transaction.
func getCompleteMultipartMD5(parts []CompletePart) string { func getCompleteMultipartMD5(parts []CompletePart) string {
var finalMD5Bytes []byte var finalMD5Bytes []byte

34
cmd/object-handlers.go
View File

@ -1182,7 +1182,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10) compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
reader = etag.NewReader(reader, nil) reader = etag.NewReader(ctx, reader, nil, nil)
wantEncryption := crypto.Requested(r.Header) wantEncryption := crypto.Requested(r.Header)
s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption) s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption)
dstOpts.IndexCB = cb dstOpts.IndexCB = cb
@ -1195,7 +1195,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
reader = gr reader = gr
} }
srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualSize) srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -1316,7 +1316,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
} }
// do not try to verify encrypted content // do not try to verify encrypted content
srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", actualSize) srcInfo.Reader, err = hash.NewReader(ctx, reader, targetSize, "", "", actualSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -1743,7 +1743,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10) metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize) actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -1763,8 +1763,20 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
md5hex = "" // Do not try to verify the content. md5hex = "" // Do not try to verify the content.
sha256hex = "" sha256hex = ""
} }
var hashReader *hash.Reader
hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize) // Optimization: If SSE-KMS and SSE-C did not request Content-Md5. Use uuid as etag
if !etag.ContentMD5Requested(r.Header) && (crypto.S3KMS.IsRequested(r.Header) || crypto.SSEC.IsRequested(r.Header)) {
hashReader, err = hash.NewReaderWithOpts(ctx, reader, hash.Options{
Size: size,
MD5Hex: md5hex,
SHA256Hex: sha256hex,
ActualSize: actualSize,
DisableMD5: false,
ForceMD5: mustGetUUIDBytes(),
})
} else {
hashReader, err = hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
}
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -1860,7 +1872,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
} }
// do not try to verify encrypted content // do not try to verify encrypted content
hashReader, err = hash.NewReader(etag.Wrap(reader, hashReader), wantSize, "", "", actualSize) hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -2077,7 +2089,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
} }
} }
hreader, err := hash.NewReader(reader, size, md5hex, sha256hex, size) hreader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, size)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -2128,7 +2140,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10) metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
actualReader, err := hash.NewReader(reader, size, "", "", actualSize) actualReader, err := hash.NewReader(ctx, reader, size, "", "", actualSize)
if err != nil { if err != nil {
return err return err
} }
@ -2142,7 +2154,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
size = -1 // Since compressed size is un-predictable. size = -1 // Since compressed size is un-predictable.
} }
hashReader, err := hash.NewReader(reader, size, "", "", actualSize) hashReader, err := hash.NewReader(ctx, reader, size, "", "", actualSize)
if err != nil { if err != nil {
return err return err
} }
@ -2212,7 +2224,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
} }
// do not try to verify encrypted content // do not try to verify encrypted content
hashReader, err = hash.NewReader(etag.Wrap(reader, hashReader), wantSize, "", "", actualSize) hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
if err != nil { if err != nil {
return err return err
} }

12
cmd/object-multipart-handlers.go
View File

@ -448,7 +448,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
} }
actualPartSize = length actualPartSize = length
var reader io.Reader = etag.NewReader(gr, nil) var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts) mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
if err != nil { if err != nil {
@ -471,7 +471,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
length = -1 length = -1
} }
srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualPartSize) srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -526,7 +526,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
wantSize = info.EncryptedSize() wantSize = info.EncryptedSize()
} }
srcInfo.Reader, err = hash.NewReader(reader, wantSize, "", "", actualPartSize) srcInfo.Reader, err = hash.NewReader(ctx, reader, wantSize, "", "", actualPartSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -715,7 +715,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"] _, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
var idxCb func() []byte var idxCb func() []byte
if isCompressed { if isCompressed {
actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize) actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -736,7 +736,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
sha256hex = "" sha256hex = ""
} }
hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize) hashReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -798,7 +798,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
wantSize = info.EncryptedSize() wantSize = info.EncryptedSize()
} }
// do not try to verify encrypted content // do not try to verify encrypted content
hashReader, err = hash.NewReader(etag.Wrap(reader, hashReader), wantSize, "", "", actualSize) hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return

4
cmd/post-policy-fan-out.go
View File

@ -61,7 +61,7 @@ func fanOutPutObject(ctx context.Context, bucket string, objectAPI ObjectLayer,
ActualSize: -1, ActualSize: -1,
DisableMD5: true, DisableMD5: true,
} }
hr, err := hash.NewReaderWithOpts(bytes.NewReader(fanOutBuf), hopts) hr, err := hash.NewReaderWithOpts(ctx, bytes.NewReader(fanOutBuf), hopts)
if err != nil { if err != nil {
errs[idx] = err errs[idx] = err
return return
@ -91,7 +91,7 @@ func fanOutPutObject(ctx context.Context, bucket string, objectAPI ObjectLayer,
} }
// do not try to verify encrypted content/ // do not try to verify encrypted content/
hr, err = hash.NewReader(encrd, -1, "", "", -1) hr, err = hash.NewReader(ctx, encrd, -1, "", "", -1)
if err != nil { if err != nil {
errs[idx] = err errs[idx] = err
return return

2
cmd/test-utils_test.go
View File

@ -162,7 +162,7 @@ func calculateSignedChunkLength(chunkDataSize int64) int64 {
} }
func mustGetPutObjReader(t TestErrHandler, data io.Reader, size int64, md5hex, sha256hex string) *PutObjReader { func mustGetPutObjReader(t TestErrHandler, data io.Reader, size int64, md5hex, sha256hex string) *PutObjReader {
hr, err := hash.NewReader(data, size, md5hex, sha256hex, size) hr, err := hash.NewReader(context.Background(), data, size, md5hex, sha256hex, size)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

8
cmd/tier.go
View File

@ -287,7 +287,7 @@ func (config *TierConfigMgr) getDriver(tierName string) (d WarmBackend, err erro
// using a PutObject API. PutObjReader encrypts json encoded tier configurations // using a PutObject API. PutObjReader encrypts json encoded tier configurations
// if KMS is enabled, otherwise simply yields the json encoded bytes as is. // if KMS is enabled, otherwise simply yields the json encoded bytes as is.
// Similarly, ObjectOptions value depends on KMS' status. // Similarly, ObjectOptions value depends on KMS' status.
func (config *TierConfigMgr) configReader() (*PutObjReader, *ObjectOptions, error) { func (config *TierConfigMgr) configReader(ctx context.Context) (*PutObjReader, *ObjectOptions, error) {
b, err := config.Bytes() b, err := config.Bytes()
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
@ -295,7 +295,7 @@ func (config *TierConfigMgr) configReader() (*PutObjReader, *ObjectOptions, erro
payloadSize := int64(len(b)) payloadSize := int64(len(b))
br := bytes.NewReader(b) br := bytes.NewReader(b)
hr, err := hash.NewReader(br, payloadSize, "", "", payloadSize) hr, err := hash.NewReader(ctx, br, payloadSize, "", "", payloadSize)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -318,7 +318,7 @@ func (config *TierConfigMgr) configReader() (*PutObjReader, *ObjectOptions, erro
Size: payloadSize, Size: payloadSize,
} }
encSize := info.EncryptedSize() encSize := info.EncryptedSize()
encHr, err := hash.NewReader(encBr, encSize, "", "", encSize) encHr, err := hash.NewReader(ctx, encBr, encSize, "", "", encSize)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -371,7 +371,7 @@ func (config *TierConfigMgr) Save(ctx context.Context, objAPI ObjectLayer) error
return errServerNotInitialized return errServerNotInitialized
} }
pr, opts, err := globalTierConfigMgr.configReader() pr, opts, err := globalTierConfigMgr.configReader(ctx)
if err != nil { if err != nil {
return err return err
} }

7
internal/etag/etag.go
View File

@ -119,6 +119,7 @@ import (
"github.com/minio/minio/internal/fips" "github.com/minio/minio/internal/fips"
"github.com/minio/minio/internal/hash/sha256" "github.com/minio/minio/internal/hash/sha256"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/sio" "github.com/minio/sio"
) )
@ -263,6 +264,12 @@ func FromContentMD5(h http.Header) (ETag, error) {
return ETag(b), nil return ETag(b), nil
} }
// ContentMD5Requested - for http.request.header is not request Content-Md5
func ContentMD5Requested(h http.Header) bool {
_, ok := h[xhttp.ContentMD5]
return ok
}
// Multipart computes an S3 multipart ETag given a list of // Multipart computes an S3 multipart ETag given a list of
// S3 singlepart ETags. It returns nil if the list of // S3 singlepart ETags. It returns nil if the list of
// ETags is empty. // ETags is empty.

3
internal/etag/etag_test.go
View File

@ -18,6 +18,7 @@
package etag package etag
import ( import (
"context"
"io" "io"
"net/http" "net/http"
"strings" "strings"
@ -137,7 +138,7 @@ var readerTests = []struct { // Reference values computed by: echo <content> | m
func TestReader(t *testing.T) { func TestReader(t *testing.T) {
for i, test := range readerTests { for i, test := range readerTests {
reader := NewReader(strings.NewReader(test.Content), test.ETag) reader := NewReader(context.Background(), strings.NewReader(test.Content), test.ETag, nil)
if _, err := io.Copy(io.Discard, reader); err != nil { if _, err := io.Copy(io.Discard, reader); err != nil {
t.Fatalf("Test %d: read failed: %v", i, err) t.Fatalf("Test %d: read failed: %v", i, err)
} }

47
internal/etag/reader.go
View File

@ -18,6 +18,7 @@
package etag package etag
import ( import (
"context"
"crypto/md5" "crypto/md5"
"fmt" "fmt"
"hash" "hash"
@ -102,12 +103,19 @@ type Reader struct {
// If the provided etag is not nil the returned // If the provided etag is not nil the returned
// Reader compares the etag with the computed // Reader compares the etag with the computed
// MD5 sum once the r returns io.EOF. // MD5 sum once the r returns io.EOF.
func NewReader(r io.Reader, etag ETag) *Reader { func NewReader(ctx context.Context, r io.Reader, etag ETag, forceMD5 []byte) *Reader {
if er, ok := r.(*Reader); ok { if er, ok := r.(*Reader); ok {
if er.readN == 0 && Equal(etag, er.checksum) { if er.readN == 0 && Equal(etag, er.checksum) {
return er return er
} }
} }
if len(forceMD5) != 0 {
return &Reader{
src: r,
md5: NewUUIDHash(forceMD5),
checksum: etag,
}
}
return &Reader{ return &Reader{
src: r, src: r,
md5: md5.New(), md5: md5.New(),
@ -153,3 +161,40 @@ type VerifyError struct {
func (v VerifyError) Error() string { func (v VerifyError) Error() string {
return fmt.Sprintf("etag: expected ETag %q does not match computed ETag %q", v.Expected, v.Computed) return fmt.Sprintf("etag: expected ETag %q does not match computed ETag %q", v.Expected, v.Computed)
} }
// UUIDHash - use uuid to make md5sum
type UUIDHash struct {
uuid []byte
}
// Write - implement hash.Hash Write
func (u UUIDHash) Write(p []byte) (n int, err error) {
return len(p), nil
}
// Sum - implement md5.Sum
func (u UUIDHash) Sum(b []byte) []byte {
return u.uuid
}
// Reset - implement hash.Hash Reset
func (u UUIDHash) Reset() {
return
}
// Size - implement hash.Hash Size
func (u UUIDHash) Size() int {
return len(u.uuid)
}
// BlockSize - implement hash.Hash BlockSize
func (u UUIDHash) BlockSize() int {
return md5.BlockSize
}
var _ hash.Hash = &UUIDHash{}
// NewUUIDHash - new UUIDHash
func NewUUIDHash(uuid []byte) *UUIDHash {
return &UUIDHash{uuid: uuid}
}

16
internal/hash/reader.go
View File

@ -19,6 +19,7 @@ package hash
import ( import (
"bytes" "bytes"
"context"
"encoding/base64" "encoding/base64"
"encoding/hex" "encoding/hex"
"errors" "errors"
@ -71,13 +72,14 @@ type Options struct {
Size int64 Size int64
ActualSize int64 ActualSize int64
DisableMD5 bool DisableMD5 bool
ForceMD5 []byte
} }
// NewReaderWithOpts is like NewReader but takes `Options` as argument, allowing // NewReaderWithOpts is like NewReader but takes `Options` as argument, allowing
// callers to indicate if they want to disable md5sum checksum. // callers to indicate if they want to disable md5sum checksum.
func NewReaderWithOpts(src io.Reader, opts Options) (*Reader, error) { func NewReaderWithOpts(ctx context.Context, src io.Reader, opts Options) (*Reader, error) {
// return hard limited reader // return hard limited reader
return newReader(src, opts.Size, opts.MD5Hex, opts.SHA256Hex, opts.ActualSize, opts.DisableMD5) return newReader(ctx, src, opts.Size, opts.MD5Hex, opts.SHA256Hex, opts.ActualSize, opts.DisableMD5, opts.ForceMD5)
} }
// NewReader returns a new Reader that wraps src and computes // NewReader returns a new Reader that wraps src and computes
@ -95,11 +97,11 @@ func NewReaderWithOpts(src io.Reader, opts Options) (*Reader, error) {
// checksums multiple times. // checksums multiple times.
// NewReader enforces S3 compatibility strictly by ensuring caller // NewReader enforces S3 compatibility strictly by ensuring caller
// does not send more content than specified size. // does not send more content than specified size.
func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) { func NewReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) {
return newReader(src, size, md5Hex, sha256Hex, actualSize, false) return newReader(ctx, src, size, md5Hex, sha256Hex, actualSize, false, nil)
} }
func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64, disableMD5 bool) (*Reader, error) { func newReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64, disableMD5 bool, forceMD5 []byte) (*Reader, error) {
MD5, err := hex.DecodeString(md5Hex) MD5, err := hex.DecodeString(md5Hex)
if err != nil { if err != nil {
return nil, BadDigest{ // TODO(aead): Return an error that indicates that an invalid ETag has been specified return nil, BadDigest{ // TODO(aead): Return an error that indicates that an invalid ETag has been specified
@ -153,7 +155,7 @@ func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
r := ioutil.HardLimitReader(src, size) r := ioutil.HardLimitReader(src, size)
if !disableMD5 { if !disableMD5 {
if _, ok := src.(etag.Tagger); !ok { if _, ok := src.(etag.Tagger); !ok {
src = etag.NewReader(r, MD5) src = etag.NewReader(ctx, r, MD5, forceMD5)
} else { } else {
src = etag.Wrap(r, src) src = etag.Wrap(r, src)
} }
@ -162,7 +164,7 @@ func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
} }
} else if _, ok := src.(etag.Tagger); !ok { } else if _, ok := src.(etag.Tagger); !ok {
if !disableMD5 { if !disableMD5 {
src = etag.NewReader(src, MD5) src = etag.NewReader(ctx, src, MD5, forceMD5)
} }
} }
var h hash.Hash var h hash.Hash

9
internal/hash/reader_test.go
View File

@ -19,6 +19,7 @@ package hash
import ( import (
"bytes" "bytes"
"context"
"encoding/base64" "encoding/base64"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
@ -30,7 +31,7 @@ import (
// Tests functions like Size(), MD5*(), SHA256*() // Tests functions like Size(), MD5*(), SHA256*()
func TestHashReaderHelperMethods(t *testing.T) { func TestHashReaderHelperMethods(t *testing.T) {
r, err := NewReader(bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4) r, err := NewReader(context.Background(), bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -194,7 +195,7 @@ func TestHashReaderVerification(t *testing.T) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) { t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
r, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize) r, err := NewReader(context.Background(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
if err != nil { if err != nil {
t.Fatalf("Test %q: Initializing reader failed %s", testCase.desc, err) t.Fatalf("Test %q: Initializing reader failed %s", testCase.desc, err)
} }
@ -213,7 +214,7 @@ func TestHashReaderVerification(t *testing.T) {
} }
func mustReader(t *testing.T, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) *Reader { func mustReader(t *testing.T, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) *Reader {
r, err := NewReader(src, size, md5Hex, sha256Hex, actualSize) r, err := NewReader(context.Background(), src, size, md5Hex, sha256Hex, actualSize)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -303,7 +304,7 @@ func TestHashReaderInvalidArguments(t *testing.T) {
for i, testCase := range testCases { for i, testCase := range testCases {
t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) { t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
_, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize) _, err := NewReader(context.Background(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
if err != nil && testCase.success { if err != nil && testCase.success {
t.Errorf("Test %q: Expected success, but got error %s instead", testCase.desc, err) t.Errorf("Test %q: Expected success, but got error %s instead", testCase.desc, err)
} }