MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 05:34:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add Full Object Checksums and CRC64-NVME (#20855)

Browse Source 
Backport of AIStor PR 247.

Add support for full object checksums as described here:

https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html

New checksum types are fully supported. Mint tests from https://github.com/minio/minio-go/pull/2026 are now passing.

Includes fixes from https://github.com/minio/minio/pull/20743 for mint tests.

Add using checksums as validation for object content. Fixes #20845 #20849

Fixes checksum replication (downstream PR 250)
This commit is contained in:
Klaus Post
2025-01-20 06:49:07 -08:00
committed by GitHub
parent 779ec8f0d4
commit 827004cd6d
18 changed files with 665 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
cmd/api-response.go
View File

@@ -166,10 +166,11 @@ type Part struct {
Size int64
// Checksum values
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
}
// ListPartsResponse - format for list parts response.
@@ -192,6 +193,8 @@ type ListPartsResponse struct {
IsTruncated bool
ChecksumAlgorithm string
ChecksumType string
// List of parts.
Parts []Part `xml:"Part"`
}
@@ -413,10 +416,11 @@ type CompleteMultipartUploadResponse struct {
Key string
ETag string
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
}
// DeleteError structure.
@@ -793,11 +797,12 @@ func generateCompleteMultipartUploadResponse(bucket, key, location string, oi Ob
Bucket: bucket,
Key: key,
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
ETag: "\"" + oi.ETag + "\"",
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
ETag: "\"" + oi.ETag + "\"",
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
}
return c
}
@@ -825,6 +830,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
listPartsResponse.IsTruncated = partsInfo.IsTruncated
listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
listPartsResponse.ChecksumType = partsInfo.ChecksumType
listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
for index, part := range partsInfo.Parts {
@@ -837,6 +843,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
newPart.ChecksumCRC32C = part.ChecksumCRC32C
newPart.ChecksumSHA1 = part.ChecksumSHA1
newPart.ChecksumSHA256 = part.ChecksumSHA256
newPart.ChecksumCRC64NVME = part.ChecksumCRC64NVME
listPartsResponse.Parts[index] = newPart
}
return listPartsResponse