MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove checksums from HTTP range request, add part checksums (#17105)

This commit is contained in:
Klaus Post 2023-04-28 08:26:32 -07:00 committed by GitHub
parent 6e27264c6b
commit 7fad0c8b41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 57 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/api-response.go
View File

@ -743,7 +743,7 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
cs := oi.decryptChecksums()
cs := oi.decryptChecksums(0)
c := CompleteMultipartUploadResponse{
Location: location,
Bucket: bucket,

5
cmd/encryption-v1.go
View File

@ -1124,7 +1124,8 @@ func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn
}
// decryptChecksums will attempt to decode checksums and return it/them if set.
func (o *ObjectInfo) decryptChecksums() map[string]string {
// if part > 0, and we have the checksum for the part that will be returned.
func (o *ObjectInfo) decryptChecksums(part int) map[string]string {
data := o.Checksum
if len(data) == 0 {
return nil
@ -1137,5 +1138,5 @@ func (o *ObjectInfo) decryptChecksums() map[string]string {
}
data = decrypted
}
return hash.ReadCheckSums(data)
return hash.ReadCheckSums(data, part)
}

4
cmd/erasure-multipart.go
View File

@ -1135,9 +1135,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
}
}
if checksumType.IsSet() {
checksumType |= hash.ChecksumMultipart
checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
cs := hash.NewChecksumFromData(checksumType, checksumCombined)
fi.Checksum = cs.AppendTo(nil, len(fi.Parts))
fi.Checksum = cs.AppendTo(nil, checksumCombined)
if opts.EncryptFn != nil {
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
}

2
cmd/erasure-object.go
View File

@ -1086,7 +1086,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
}
fi.DataDir = mustGetUUID()
fi.Checksum = opts.WantChecksum.AppendTo(nil, 0)
fi.Checksum = opts.WantChecksum.AppendTo(nil, nil)
if opts.EncryptFn != nil {
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
}

2
cmd/object-handlers-common.go
View File

@ -340,7 +340,7 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
}
}
hash.AddChecksumHeader(w, objInfo.decryptChecksums())
hash.AddChecksumHeader(w, objInfo.decryptChecksums(0))
}
func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toDel []ObjectToDelete) {

10
cmd/object-handlers.go
View File

@ -513,8 +513,9 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
}
if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" {
hash.AddChecksumHeader(w, objInfo.decryptChecksums())
if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil {
// AWS S3 silently drops checksums on range requests.
hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber))
}
if err = setObjectHeaders(w, objInfo, rs, opts); err != nil {
@ -800,8 +801,9 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
}
if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" {
hash.AddChecksumHeader(w, objInfo.decryptChecksums())
if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil {
// AWS S3 silently drops checksums on range requests.
hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber))
}
// Set standard object headers.

49
internal/hash/checksum.go
View File

@ -19,6 +19,7 @@ package hash
import (
"bytes"
"context"
"crypto/sha1"
"encoding/base64"
"encoding/binary"
@ -30,6 +31,7 @@ import (
"github.com/minio/minio/internal/hash/sha256"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
)
// MinIOMultipartChecksum is as metadata on multipart uploads to indicate checksum type.
@ -56,6 +58,8 @@ const (
ChecksumInvalid
// ChecksumMultipart indicates the checksum is from a multipart upload.
ChecksumMultipart
// ChecksumIncludesMultipart indicates the checksum also contains part checksums.
ChecksumIncludesMultipart
// ChecksumNone indicates no checksum.
ChecksumNone ChecksumType = 0
@ -183,7 +187,7 @@ func NewChecksumFromData(t ChecksumType, data []byte) *Checksum {
}
// ReadCheckSums will read checksums from b and return them.
func ReadCheckSums(b []byte) map[string]string {
func ReadCheckSums(b []byte, part int) map[string]string {
res := make(map[string]string, 1)
for len(b) > 0 {
t, n := binary.Uvarint(b)
@ -206,8 +210,29 @@ func ReadCheckSums(b []byte) map[string]string {
}
cs = fmt.Sprintf("%s-%d", cs, t)
b = b[n:]
if part > 0 {
cs = ""
}
if typ.Is(ChecksumIncludesMultipart) {
wantLen := int(t) * length
if len(b) < wantLen {
break
}
// Read part checksum
if part > 0 && uint64(part) <= t {
offset := (part - 1) * length
partCs := b[offset:]
cs = base64.StdEncoding.EncodeToString(partCs[:length])
}
b = b[wantLen:]
}
} else if part > 1 {
// For non-multipart, checksum is part 1.
cs = ""
}
if cs != "" {
res[typ.String()] = cs
}
res[typ.String()] = cs
}
if len(res) == 0 {
res = nil
@ -239,7 +264,7 @@ func NewChecksumString(alg, value string) *Checksum {
// AppendTo will append the checksum to b.
// 'parts' is used when checksum has ChecksumMultipart set.
// ReadCheckSums reads the values back.
func (c *Checksum) AppendTo(b []byte, parts int) []byte {
func (c *Checksum) AppendTo(b []byte, parts []byte) []byte {
if c == nil {
return nil
}
@ -252,11 +277,23 @@ func (c *Checksum) AppendTo(b []byte, parts int) []byte {
b = append(b, tmp[:n]...)
b = append(b, crc...)
if c.Type.Is(ChecksumMultipart) {
if parts < 0 {
parts = 0
var checksums int
// Ensure we don't divide by 0:
if c.Type.RawByteLen() == 0 || len(parts)%c.Type.RawByteLen() != 0 {
logger.LogIf(context.Background(), fmt.Errorf("internal error: Unexpected checksum length: %d, each checksum %d", len(parts), c.Type.RawByteLen()))
checksums = 0
parts = nil
} else {
checksums = len(parts) / c.Type.RawByteLen()
}
n := binary.PutUvarint(tmp[:], uint64(parts))
if !c.Type.Is(ChecksumIncludesMultipart) {
parts = nil
}
n := binary.PutUvarint(tmp[:], uint64(checksums))
b = append(b, tmp[:n]...)
if len(parts) > 0 {
b = append(b, parts...)
}
}
return b
}