MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add Full Object Checksums and CRC64-NVME (#20855)

Browse Source 
Backport of AIStor PR 247.

Add support for full object checksums as described here:

https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html

New checksum types are fully supported. Mint tests from https://github.com/minio/minio-go/pull/2026 are now passing.

Includes fixes from https://github.com/minio/minio/pull/20743 for mint tests.

Add using checksums as validation for object content. Fixes #20845 #20849

Fixes checksum replication (downstream PR 250)
This commit is contained in:
Klaus Post
2025-01-20 06:49:07 -08:00
committed by GitHub
parent 779ec8f0d4
commit 827004cd6d
18 changed files with 665 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/api-errors.go
View File

@@ -629,7 +629,7 @@ var errorCodes = errorCodeMap{
},
ErrMissingContentMD5: {
Code: "MissingContentMD5",
Description: "Missing required header for this request: Content-Md5.",
Description: "Missing or invalid required header for this request: Content-Md5 or Amz-Content-Checksum",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingSecurityHeader: {

33
cmd/api-response.go
View File

@@ -166,10 +166,11 @@ type Part struct {
Size int64
// Checksum values
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
}
// ListPartsResponse - format for list parts response.
@@ -192,6 +193,8 @@ type ListPartsResponse struct {
IsTruncated bool
ChecksumAlgorithm string
ChecksumType string
// List of parts.
Parts []Part `xml:"Part"`
}
@@ -413,10 +416,11 @@ type CompleteMultipartUploadResponse struct {
Key string
ETag string
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
}
// DeleteError structure.
@@ -793,11 +797,12 @@ func generateCompleteMultipartUploadResponse(bucket, key, location string, oi Ob
Bucket: bucket,
Key: key,
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
ETag: "\"" + oi.ETag + "\"",
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
ETag: "\"" + oi.ETag + "\"",
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
}
return c
}
@@ -825,6 +830,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
listPartsResponse.IsTruncated = partsInfo.IsTruncated
listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
listPartsResponse.ChecksumType = partsInfo.ChecksumType
listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
for index, part := range partsInfo.Parts {
@@ -837,6 +843,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
newPart.ChecksumCRC32C = part.ChecksumCRC32C
newPart.ChecksumSHA1 = part.ChecksumSHA1
newPart.ChecksumSHA256 = part.ChecksumSHA256
newPart.ChecksumCRC64NVME = part.ChecksumCRC64NVME
listPartsResponse.Parts[index] = newPart
}
return listPartsResponse

2
cmd/bucket-handlers.go
View File

@@ -429,7 +429,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
// Content-Md5 is required should be set
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if _, ok := r.Header[xhttp.ContentMD5]; !ok {
if !validateLengthAndChecksum(r) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
return
}

5
cmd/bucket-lifecycle-handlers.go
View File

@@ -19,7 +19,6 @@ package cmd
import (
"encoding/xml"
"io"
"net/http"
"strconv"
"time"
@@ -53,7 +52,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
bucket := vars["bucket"]
// PutBucketLifecycle always needs a Content-Md5
if _, ok := r.Header[xhttp.ContentMD5]; !ok {
if !validateLengthAndChecksum(r) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
return
}
@@ -70,7 +69,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
return
}
bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(io.LimitReader(r.Body, r.ContentLength))
bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(r.Body)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return

52
cmd/bucket-replication.go
View File

@@ -794,18 +794,23 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
meta[k] = v
}
}
if len(objInfo.Checksum) > 0 {
// Add encrypted CRC to metadata for SSE-C objects.
if isSSEC {
meta[ReplicationSsecChecksumHeader] = base64.StdEncoding.EncodeToString(objInfo.Checksum)
} else {
for _, pi := range objInfo.Parts {
if pi.Number == partNum {
for k, v := range pi.Checksums {
meta[k] = v
if objInfo.isMultipart() && partNum > 0 {
for _, pi := range objInfo.Parts {
if pi.Number == partNum {
for k, v := range pi.Checksums { // for PutObjectPart
meta[k] = v
}
}
}
} else {
for k, v := range getCRCMeta(objInfo, 0, nil) { // for PutObject/NewMultipartUpload
meta[k] = v
}
}
}
}
@@ -1666,7 +1671,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
cHeader := http.Header{}
cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
if !isSSEC {
for k, v := range partInfo.Checksums {
for k, v := range getCRCMeta(objInfo, partInfo.Number, nil) {
cHeader.Add(k, v)
}
}
@@ -1690,12 +1695,13 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
return fmt.Errorf("ssec(%t): Part size mismatch: got %d, want %d", isSSEC, pInfo.Size, size)
}
uploadedParts = append(uploadedParts, minio.CompletePart{
PartNumber: pInfo.PartNumber,
ETag: pInfo.ETag,
ChecksumCRC32: pInfo.ChecksumCRC32,
ChecksumCRC32C: pInfo.ChecksumCRC32C,
ChecksumSHA1: pInfo.ChecksumSHA1,
ChecksumSHA256: pInfo.ChecksumSHA256,
PartNumber: pInfo.PartNumber,
ETag: pInfo.ETag,
ChecksumCRC32: pInfo.ChecksumCRC32,
ChecksumCRC32C: pInfo.ChecksumCRC32C,
ChecksumSHA1: pInfo.ChecksumSHA1,
ChecksumSHA256: pInfo.ChecksumSHA256,
ChecksumCRC64NVME: pInfo.ChecksumCRC64NVME,
})
}
userMeta := map[string]string{
@@ -1708,6 +1714,12 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
// really big value but its okay on heavily loaded systems. This is just tail end timeout.
cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
defer ccancel()
if len(objInfo.Checksum) > 0 {
for k, v := range getCRCMeta(objInfo, 0, nil) {
userMeta[k] = v
}
}
_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
UserMetadata: userMeta,
Internal: minio.AdvancedPutOptions{
@@ -3753,3 +3765,19 @@ type validateReplicationDestinationOptions struct {
checkReadyErr sync.Map
}
func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) map[string]string {
meta := make(map[string]string)
cs := oi.decryptChecksums(partNum, h)
for k, v := range cs {
cksum := hash.NewChecksumString(k, v)
if cksum == nil {
continue
}
if cksum.Valid() {
meta[cksum.Type.Key()] = v
}
meta[xhttp.AmzChecksumType] = cksum.Type.ObjType()
}
return meta
}

94
cmd/erasure-multipart.go
View File

@@ -480,6 +480,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
if opts.WantChecksum != nil && opts.WantChecksum.Type.IsSet() {
userDefined[hash.MinIOMultipartChecksum] = opts.WantChecksum.Type.String()
userDefined[hash.MinIOMultipartChecksumType] = opts.WantChecksum.Type.ObjType()
}
modTime := opts.MTime
@@ -508,6 +509,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
return &NewMultipartUploadResult{
UploadID: uploadID,
ChecksumAlgo: userDefined[hash.MinIOMultipartChecksum],
ChecksumType: userDefined[hash.MinIOMultipartChecksumType],
}, nil
}
@@ -765,15 +767,16 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
// Return success.
return PartInfo{
PartNumber: partInfo.Number,
ETag: partInfo.ETag,
LastModified: partInfo.ModTime,
Size: partInfo.Size,
ActualSize: partInfo.ActualSize,
ChecksumCRC32: partInfo.Checksums["CRC32"],
ChecksumCRC32C: partInfo.Checksums["CRC32C"],
ChecksumSHA1: partInfo.Checksums["SHA1"],
ChecksumSHA256: partInfo.Checksums["SHA256"],
PartNumber: partInfo.Number,
ETag: partInfo.ETag,
LastModified: partInfo.ModTime,
Size: partInfo.Size,
ActualSize: partInfo.ActualSize,
ChecksumCRC32: partInfo.Checksums["CRC32"],
ChecksumCRC32C: partInfo.Checksums["CRC32C"],
ChecksumSHA1: partInfo.Checksums["SHA1"],
ChecksumSHA256: partInfo.Checksums["SHA256"],
ChecksumCRC64NVME: partInfo.Checksums["CRC64NVME"],
}, nil
}
@@ -895,6 +898,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
result.PartNumberMarker = partNumberMarker
result.UserDefined = cloneMSS(fi.Metadata)
result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum]
result.ChecksumType = fi.Metadata[hash.MinIOMultipartChecksumType]
if maxParts == 0 {
return result, nil
@@ -941,15 +945,16 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
count := maxParts
for _, objPart := range objParts {
result.Parts = append(result.Parts, PartInfo{
PartNumber: objPart.Number,
LastModified: objPart.ModTime,
ETag: objPart.ETag,
Size: objPart.Size,
ActualSize: objPart.ActualSize,
ChecksumCRC32: objPart.Checksums["CRC32"],
ChecksumCRC32C: objPart.Checksums["CRC32C"],
ChecksumSHA1: objPart.Checksums["SHA1"],
ChecksumSHA256: objPart.Checksums["SHA256"],
PartNumber: objPart.Number,
LastModified: objPart.ModTime,
ETag: objPart.ETag,
Size: objPart.Size,
ActualSize: objPart.ActualSize,
ChecksumCRC32: objPart.Checksums["CRC32"],
ChecksumCRC32C: objPart.Checksums["CRC32C"],
ChecksumSHA1: objPart.Checksums["SHA1"],
ChecksumSHA256: objPart.Checksums["SHA256"],
ChecksumCRC64NVME: objPart.Checksums["CRC64NVME"],
})
count--
if count == 0 {
@@ -1131,12 +1136,12 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
// Checksum type set when upload started.
var checksumType hash.ChecksumType
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
checksumType = hash.NewChecksumType(cs)
checksumType = hash.NewChecksumType(cs, fi.Metadata[hash.MinIOMultipartChecksumType])
if opts.WantChecksum != nil && !opts.WantChecksum.Type.Is(checksumType) {
return oi, InvalidArgument{
Bucket: bucket,
Object: fi.Name,
Err: fmt.Errorf("checksum type mismatch"),
Err: fmt.Errorf("checksum type mismatch. got %q (%s) expected %q (%s)", checksumType.String(), checksumType.ObjType(), opts.WantChecksum.Type.String(), opts.WantChecksum.Type.ObjType()),
}
}
}
@@ -1216,6 +1221,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
// Allocate parts similar to incoming slice.
fi.Parts = make([]ObjectPartInfo, len(parts))
var checksum hash.Checksum
checksum.Type = checksumType
// Validate each part and then commit to disk.
for i, part := range parts {
partIdx := objectPartIndex(currentFI.Parts, part.PartNumber)
@@ -1249,10 +1257,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
}
}
wantCS := map[string]string{
hash.ChecksumCRC32.String(): part.ChecksumCRC32,
hash.ChecksumCRC32C.String(): part.ChecksumCRC32C,
hash.ChecksumSHA1.String(): part.ChecksumSHA1,
hash.ChecksumSHA256.String(): part.ChecksumSHA256,
hash.ChecksumCRC32.String(): part.ChecksumCRC32,
hash.ChecksumCRC32C.String(): part.ChecksumCRC32C,
hash.ChecksumSHA1.String(): part.ChecksumSHA1,
hash.ChecksumSHA256.String(): part.ChecksumSHA256,
hash.ChecksumCRC64NVME.String(): part.ChecksumCRC64NVME,
}
if wantCS[checksumType.String()] != crc {
return oi, InvalidPart{
@@ -1267,6 +1276,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
PartNumber: part.PartNumber,
}
}
if checksumType.FullObjectRequested() {
if err := checksum.AddPart(*cs, expPart.ActualSize); err != nil {
return oi, InvalidPart{
PartNumber: part.PartNumber,
ExpETag: "<nil>",
GotETag: err.Error(),
}
}
}
checksumCombined = append(checksumCombined, cs.Raw...)
}
@@ -1297,9 +1315,19 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
}
if opts.WantChecksum != nil {
err := opts.WantChecksum.Matches(checksumCombined, len(parts))
if err != nil {
return oi, err
if checksumType.FullObjectRequested() {
if opts.WantChecksum.Encoded != checksum.Encoded {
err := hash.ChecksumMismatch{
Want: opts.WantChecksum.Encoded,
Got: checksum.Encoded,
}
return oi, err
}
} else {
err := opts.WantChecksum.Matches(checksumCombined, len(parts))
if err != nil {
return oi, err
}
}
}
@@ -1313,14 +1341,18 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
if checksumType.IsSet() {
checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
var cs *hash.Checksum
cs = hash.NewChecksumFromData(checksumType, checksumCombined)
fi.Checksum = cs.AppendTo(nil, checksumCombined)
checksum.Type = checksumType
if !checksumType.FullObjectRequested() {
checksum = *hash.NewChecksumFromData(checksumType, checksumCombined)
}
fi.Checksum = checksum.AppendTo(nil, checksumCombined)
if opts.EncryptFn != nil {
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
}
}
delete(fi.Metadata, hash.MinIOMultipartChecksum) // Not needed in final object.
// Remove superfluous internal headers.
delete(fi.Metadata, hash.MinIOMultipartChecksum)
delete(fi.Metadata, hash.MinIOMultipartChecksumType)
// Save the final object size and modtime.
fi.Size = objectSize

46
cmd/object-api-datatypes.go
View File

@@ -419,6 +419,9 @@ type ListPartsInfo struct {
// ChecksumAlgorithm if set
ChecksumAlgorithm string
// ChecksumType if set
ChecksumType string
}
// Lookup - returns if uploadID is valid
@@ -597,10 +600,11 @@ type PartInfo struct {
ActualSize int64
// Checksum values
ChecksumCRC32 string
ChecksumCRC32C string
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC32 string
ChecksumCRC32C string
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
}
// CompletePart - represents the part that was completed, this is sent by the client
@@ -613,11 +617,14 @@ type CompletePart struct {
// Entity tag returned when the part was uploaded.
ETag string
Size int64
// Checksum values. Optional.
ChecksumCRC32 string
ChecksumCRC32C string
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC32 string
ChecksumCRC32C string
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
}
// CompleteMultipartUpload - represents list of parts which are completed, this is sent by the
@@ -630,6 +637,7 @@ type CompleteMultipartUpload struct {
type NewMultipartUploadResult struct {
UploadID string
ChecksumAlgo string
ChecksumType string
}
type getObjectAttributesResponse struct {
@@ -641,10 +649,11 @@ type getObjectAttributesResponse struct {
}
type objectAttributesChecksum struct {
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
}
type objectAttributesParts struct {
@@ -657,12 +666,13 @@ type objectAttributesParts struct {
}
type objectAttributesPart struct {
PartNumber int
Size int64
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
PartNumber int
Size int64
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
}
type objectAttributesErrorResponse struct {

69
cmd/object-api-datatypes_gen.go
View File

@@ -201,13 +201,16 @@ func (z *CompleteMultipartUpload) Msgsize() (s int) {
// MarshalMsg implements msgp.Marshaler
func (z *CompletePart) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 6
// map header, size 8
// string "PartNumber"
o = append(o, 0x86, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
o = append(o, 0x88, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
o = msgp.AppendInt(o, z.PartNumber)
// string "ETag"
o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67)
o = msgp.AppendString(o, z.ETag)
// string "Size"
o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
o = msgp.AppendInt64(o, z.Size)
// string "ChecksumCRC32"
o = append(o, 0xad, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x33, 0x32)
o = msgp.AppendString(o, z.ChecksumCRC32)
@@ -220,6 +223,9 @@ func (z *CompletePart) MarshalMsg(b []byte) (o []byte, err error) {
// string "ChecksumSHA256"
o = append(o, 0xae, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36)
o = msgp.AppendString(o, z.ChecksumSHA256)
// string "ChecksumCRC64NVME"
o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x36, 0x34, 0x4e, 0x56, 0x4d, 0x45)
o = msgp.AppendString(o, z.ChecksumCRC64NVME)
return
}
@@ -253,6 +259,12 @@ func (z *CompletePart) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "ETag")
return
}
case "Size":
z.Size, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Size")
return
}
case "ChecksumCRC32":
z.ChecksumCRC32, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
@@ -277,6 +289,12 @@ func (z *CompletePart) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "ChecksumSHA256")
return
}
case "ChecksumCRC64NVME":
z.ChecksumCRC64NVME, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ChecksumCRC64NVME")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -291,7 +309,7 @@ func (z *CompletePart) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *CompletePart) Msgsize() (s int) {
s = 1 + 11 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256)
s = 1 + 11 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256) + 18 + msgp.StringPrefixSize + len(z.ChecksumCRC64NVME)
return
}
@@ -956,9 +974,9 @@ func (z *ListObjectsV2Info) Msgsize() (s int) {
// MarshalMsg implements msgp.Marshaler
func (z *ListPartsInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 11
// map header, size 12
// string "Bucket"
o = append(o, 0x8b, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
o = append(o, 0x8c, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
o = msgp.AppendString(o, z.Bucket)
// string "Object"
o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74)
@@ -1001,6 +1019,9 @@ func (z *ListPartsInfo) MarshalMsg(b []byte) (o []byte, err error) {
// string "ChecksumAlgorithm"
o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d)
o = msgp.AppendString(o, z.ChecksumAlgorithm)
// string "ChecksumType"
o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x54, 0x79, 0x70, 0x65)
o = msgp.AppendString(o, z.ChecksumType)
return
}
@@ -1125,6 +1146,12 @@ func (z *ListPartsInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "ChecksumAlgorithm")
return
}
case "ChecksumType":
z.ChecksumType, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ChecksumType")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -1150,7 +1177,7 @@ func (z *ListPartsInfo) Msgsize() (s int) {
s += msgp.StringPrefixSize + len(za0002) + msgp.StringPrefixSize + len(za0003)
}
}
s += 18 + msgp.StringPrefixSize + len(z.ChecksumAlgorithm)
s += 18 + msgp.StringPrefixSize + len(z.ChecksumAlgorithm) + 13 + msgp.StringPrefixSize + len(z.ChecksumType)
return
}
@@ -1279,13 +1306,16 @@ func (z *MultipartInfo) Msgsize() (s int) {
// MarshalMsg implements msgp.Marshaler
func (z NewMultipartUploadResult) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
// map header, size 3
// string "UploadID"
o = append(o, 0x82, 0xa8, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44)
o = append(o, 0x83, 0xa8, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44)
o = msgp.AppendString(o, z.UploadID)
// string "ChecksumAlgo"
o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f)
o = msgp.AppendString(o, z.ChecksumAlgo)
// string "ChecksumType"
o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x54, 0x79, 0x70, 0x65)
o = msgp.AppendString(o, z.ChecksumType)
return
}
@@ -1319,6 +1349,12 @@ func (z *NewMultipartUploadResult) UnmarshalMsg(bts []byte) (o []byte, err error
err = msgp.WrapError(err, "ChecksumAlgo")
return
}
case "ChecksumType":
z.ChecksumType, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ChecksumType")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -1333,7 +1369,7 @@ func (z *NewMultipartUploadResult) UnmarshalMsg(bts []byte) (o []byte, err error
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z NewMultipartUploadResult) Msgsize() (s int) {
s = 1 + 9 + msgp.StringPrefixSize + len(z.UploadID) + 13 + msgp.StringPrefixSize + len(z.ChecksumAlgo)
s = 1 + 9 + msgp.StringPrefixSize + len(z.UploadID) + 13 + msgp.StringPrefixSize + len(z.ChecksumAlgo) + 13 + msgp.StringPrefixSize + len(z.ChecksumType)
return
}
@@ -1772,9 +1808,9 @@ func (z *ObjectInfo) Msgsize() (s int) {
// MarshalMsg implements msgp.Marshaler
func (z *PartInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 9
// map header, size 10
// string "PartNumber"
o = append(o, 0x89, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
o = append(o, 0x8a, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
o = msgp.AppendInt(o, z.PartNumber)
// string "LastModified"
o = append(o, 0xac, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64)
@@ -1800,6 +1836,9 @@ func (z *PartInfo) MarshalMsg(b []byte) (o []byte, err error) {
// string "ChecksumSHA256"
o = append(o, 0xae, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36)
o = msgp.AppendString(o, z.ChecksumSHA256)
// string "ChecksumCRC64NVME"
o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x36, 0x34, 0x4e, 0x56, 0x4d, 0x45)
o = msgp.AppendString(o, z.ChecksumCRC64NVME)
return
}
@@ -1875,6 +1914,12 @@ func (z *PartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "ChecksumSHA256")
return
}
case "ChecksumCRC64NVME":
z.ChecksumCRC64NVME, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ChecksumCRC64NVME")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -1889,7 +1934,7 @@ func (z *PartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *PartInfo) Msgsize() (s int) {
s = 1 + 11 + msgp.IntSize + 13 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256)
s = 1 + 11 + msgp.IntSize + 13 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256) + 18 + msgp.StringPrefixSize + len(z.ChecksumCRC64NVME)
return
}

28
cmd/object-handlers.go
View File

@@ -636,10 +636,11 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
// AWS does not appear to append part number on this API call.
if len(chkSums) > 0 {
OA.Checksum = &objectAttributesChecksum{
ChecksumCRC32: strings.Split(chkSums["CRC32"], "-")[0],
ChecksumCRC32C: strings.Split(chkSums["CRC32C"], "-")[0],
ChecksumSHA1: strings.Split(chkSums["SHA1"], "-")[0],
ChecksumSHA256: strings.Split(chkSums["SHA256"], "-")[0],
ChecksumCRC32: strings.Split(chkSums["CRC32"], "-")[0],
ChecksumCRC32C: strings.Split(chkSums["CRC32C"], "-")[0],
ChecksumSHA1: strings.Split(chkSums["SHA1"], "-")[0],
ChecksumSHA256: strings.Split(chkSums["SHA256"], "-")[0],
ChecksumCRC64NVME: strings.Split(chkSums["CRC64NVME"], "-")[0],
}
}
}
@@ -678,12 +679,13 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
OA.ObjectParts.NextPartNumberMarker = v.Number
OA.ObjectParts.Parts = append(OA.ObjectParts.Parts, &objectAttributesPart{
ChecksumSHA1: objInfo.Parts[i].Checksums["SHA1"],
ChecksumSHA256: objInfo.Parts[i].Checksums["SHA256"],
ChecksumCRC32: objInfo.Parts[i].Checksums["CRC32"],
ChecksumCRC32C: objInfo.Parts[i].Checksums["CRC32C"],
PartNumber: objInfo.Parts[i].Number,
Size: objInfo.Parts[i].Size,
ChecksumSHA1: objInfo.Parts[i].Checksums["SHA1"],
ChecksumSHA256: objInfo.Parts[i].Checksums["SHA256"],
ChecksumCRC32: objInfo.Parts[i].Checksums["CRC32"],
ChecksumCRC32C: objInfo.Parts[i].Checksums["CRC32C"],
ChecksumCRC64NVME: objInfo.Parts[i].Checksums["CRC64NVME"],
PartNumber: objInfo.Parts[i].Number,
Size: objInfo.Parts[i].Size,
})
}
}
@@ -2731,7 +2733,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if !hasContentMD5(r.Header) {
if !validateLengthAndChecksum(r) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
return
}
@@ -2741,7 +2743,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
return
}
legalHold, err := objectlock.ParseObjectLegalHold(io.LimitReader(r.Body, r.ContentLength))
legalHold, err := objectlock.ParseObjectLegalHold(r.Body)
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
apiErr.Description = err.Error()
@@ -2889,7 +2891,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
return
}
if !hasContentMD5(r.Header) {
if !validateLengthAndChecksum(r) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
return
}

5
cmd/object-multipart-handlers.go
View File

@@ -214,7 +214,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
}
}
checksumType := hash.NewChecksumType(r.Header.Get(xhttp.AmzChecksumAlgo))
checksumType := hash.NewChecksumHeader(r.Header)
if checksumType.Is(hash.ChecksumInvalid) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequestParameter), r.URL)
return
@@ -233,6 +233,9 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
response := generateInitiateMultipartUploadResponse(bucket, object, res.UploadID)
if res.ChecksumAlgo != "" {
w.Header().Set(xhttp.AmzChecksumAlgo, res.ChecksumAlgo)
if res.ChecksumType != "" {
w.Header().Set(xhttp.AmzChecksumType, res.ChecksumType)
}
}
encodedSuccessResponse := encodeResponse(response)

26
cmd/utils.go
View File

@@ -20,7 +20,9 @@ package cmd
import (
"bytes"
"context"
"crypto/md5"
"crypto/tls"
"encoding/base64"
"encoding/json"
"encoding/xml"
"errors"
@@ -254,10 +256,26 @@ func xmlDecoder(body io.Reader, v interface{}, size int64) error {
return err
}
// hasContentMD5 returns true if Content-MD5 header is set.
func hasContentMD5(h http.Header) bool {
_, ok := h[xhttp.ContentMD5]
return ok
// validateLengthAndChecksum returns if a content checksum is set,
// and will replace r.Body with a reader that checks the provided checksum
func validateLengthAndChecksum(r *http.Request) bool {
if mdFive := r.Header.Get(xhttp.ContentMD5); mdFive != "" {
want, err := base64.StdEncoding.DecodeString(mdFive)
if err != nil {
return false
}
r.Body = hash.NewChecker(r.Body, md5.New(), want, r.ContentLength)
return true
}
cs, err := hash.GetContentChecksum(r.Header)
if err != nil {
return false
}
if !cs.Type.IsSet() {
return false
}
r.Body = hash.NewChecker(r.Body, cs.Type.Hasher(), cs.Raw, r.ContentLength)
return true
}
// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html