MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-28 05:04:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add Full Object Checksums and CRC64-NVME (#20855)

Browse Source 
Backport of AIStor PR 247.

Add support for full object checksums as described here:

https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html

New checksum types are fully supported. Mint tests from https://github.com/minio/minio-go/pull/2026 are now passing.

Includes fixes from https://github.com/minio/minio/pull/20743 for mint tests.

Add using checksums as validation for object content. Fixes #20845 #20849

Fixes checksum replication (downstream PR 250)
This commit is contained in:
Klaus Post
2025-01-20 06:49:07 -08:00
committed by GitHub
parent 779ec8f0d4
commit 827004cd6d
18 changed files with 665 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
internal/hash/checksum.go
View File

@@ -26,6 +26,7 @@ import (
"fmt"
"hash"
"hash/crc32"
"hash/crc64"
"net/http"
"strconv"
"strings"
@@ -42,6 +43,9 @@ func hashLogIf(ctx context.Context, err error) {
// MinIOMultipartChecksum is as metadata on multipart uploads to indicate checksum type.
const MinIOMultipartChecksum = "x-minio-multipart-checksum"
// MinIOMultipartChecksumType is as metadata on multipart uploads to indicate checksum type.
const MinIOMultipartChecksumType = "x-minio-multipart-checksum-type"
// ChecksumType contains information about the checksum type.
type ChecksumType uint32
@@ -65,11 +69,21 @@ const (
ChecksumMultipart
// ChecksumIncludesMultipart indicates the checksum also contains part checksums.
ChecksumIncludesMultipart
// ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial.
ChecksumCRC64NVME
// ChecksumFullObject indicates the checksum is of the full object,
// not checksum of checksums. Should only be set on ChecksumMultipart
ChecksumFullObject
// ChecksumNone indicates no checksum.
ChecksumNone ChecksumType = 0
baseTypeMask = ChecksumSHA256 | ChecksumSHA1 | ChecksumCRC32 | ChecksumCRC32C | ChecksumCRC64NVME
)
// BaseChecksumTypes is a list of all the base checksum types.
var BaseChecksumTypes = []ChecksumType{ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC64NVME, ChecksumCRC32C}
// Checksum is a type and base 64 encoded value.
type Checksum struct {
Type ChecksumType
@@ -86,6 +100,11 @@ func (c ChecksumType) Is(t ChecksumType) bool {
return c&t == t
}
// Base returns the base checksum (if any)
func (c ChecksumType) Base() ChecksumType {
return c & baseTypeMask
}
// Key returns the header key.
// returns empty string if invalid or none.
func (c ChecksumType) Key() string {
@@ -98,6 +117,8 @@ func (c ChecksumType) Key() string {
return xhttp.AmzChecksumSHA1
case c.Is(ChecksumSHA256):
return xhttp.AmzChecksumSHA256
case c.Is(ChecksumCRC64NVME):
return xhttp.AmzChecksumCRC64NVME
}
return ""
}
@@ -113,32 +134,56 @@ func (c ChecksumType) RawByteLen() int {
return sha1.Size
case c.Is(ChecksumSHA256):
return sha256.Size
case c.Is(ChecksumCRC64NVME):
return crc64.Size
}
return 0
}
// IsSet returns whether the type is valid and known.
func (c ChecksumType) IsSet() bool {
return !c.Is(ChecksumInvalid) && !c.Is(ChecksumNone)
return !c.Is(ChecksumInvalid) && !c.Base().Is(ChecksumNone)
}
// NewChecksumType returns a checksum type based on the algorithm string.
func NewChecksumType(alg string) ChecksumType {
// NewChecksumType returns a checksum type based on the algorithm string and obj type.
func NewChecksumType(alg, objType string) ChecksumType {
full := ChecksumFullObject
if objType != xhttp.AmzChecksumTypeFullObject {
full = 0
}
switch strings.ToUpper(alg) {
case "CRC32":
return ChecksumCRC32
return ChecksumCRC32 | full
case "CRC32C":
return ChecksumCRC32C
return ChecksumCRC32C | full
case "SHA1":
if full != 0 {
return ChecksumInvalid
}
return ChecksumSHA1
case "SHA256":
if full != 0 {
return ChecksumInvalid
}
return ChecksumSHA256
case "CRC64NVME":
// AWS seems to ignore full value, and just assume it.
return ChecksumCRC64NVME
case "":
if full != 0 {
return ChecksumInvalid
}
return ChecksumNone
}
return ChecksumInvalid
}
// NewChecksumHeader returns a checksum type based on the algorithm string.
func NewChecksumHeader(h http.Header) ChecksumType {
return NewChecksumType(h.Get(xhttp.AmzChecksumAlgo), h.Get(xhttp.AmzChecksumType))
}
// String returns the type as a string.
func (c ChecksumType) String() string {
switch {
@@ -150,12 +195,35 @@ func (c ChecksumType) String() string {
return "SHA1"
case c.Is(ChecksumSHA256):
return "SHA256"
case c.Is(ChecksumCRC64NVME):
return "CRC64NVME"
case c.Is(ChecksumNone):
return ""
}
return "invalid"
}
// FullObjectRequested will return if the checksum type indicates full object checksum was requested.
func (c ChecksumType) FullObjectRequested() bool {
return c&(ChecksumFullObject) == ChecksumFullObject || c.Is(ChecksumCRC64NVME)
}
// ObjType returns a string to return as x-amz-checksum-type.
func (c ChecksumType) ObjType() string {
if c.FullObjectRequested() {
return xhttp.AmzChecksumTypeFullObject
}
if c.IsSet() {
return xhttp.AmzChecksumTypeComposite
}
return ""
}
// CanMerge will return if the checksum type indicates that checksums can be merged.
func (c ChecksumType) CanMerge() bool {
return c.Is(ChecksumCRC64NVME) || c.Is(ChecksumCRC32C) || c.Is(ChecksumCRC32)
}
// Hasher returns a hasher corresponding to the checksum type.
// Returns nil if no checksum.
func (c ChecksumType) Hasher() hash.Hash {
@@ -168,6 +236,8 @@ func (c ChecksumType) Hasher() hash.Hash {
return sha1.New()
case c.Is(ChecksumSHA256):
return sha256.New()
case c.Is(ChecksumCRC64NVME):
return crc64.New(crc64Table)
}
return nil
}
@@ -214,7 +284,11 @@ func ReadCheckSums(b []byte, part int) map[string]string {
if n < 0 {
break
}
cs = fmt.Sprintf("%s-%d", cs, t)
if !typ.FullObjectRequested() {
cs = fmt.Sprintf("%s-%d", cs, t)
} else if part <= 0 {
res[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
}
b = b[n:]
if part > 0 {
cs = ""
@@ -322,7 +396,7 @@ func NewChecksumWithType(alg ChecksumType, value string) *Checksum {
// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value.
func NewChecksumString(alg, value string) *Checksum {
return NewChecksumWithType(NewChecksumType(alg), value)
return NewChecksumWithType(NewChecksumType(alg, ""), value)
}
// AppendTo will append the checksum to b.
@@ -377,8 +451,7 @@ func (c Checksum) Valid() bool {
if len(c.Encoded) == 0 || c.Type.Trailing() {
return c.Type.Is(ChecksumNone) || c.Type.Trailing()
}
raw := c.Raw
return c.Type.RawByteLen() == len(raw)
return c.Type.RawByteLen() == len(c.Raw)
}
// Matches returns whether given content matches c.
@@ -440,6 +513,10 @@ func TransferChecksumHeader(w http.ResponseWriter, r *http.Request) {
// AddChecksumHeader will transfer any checksum value that has been checked.
func AddChecksumHeader(w http.ResponseWriter, c map[string]string) {
for k, v := range c {
if k == xhttp.AmzChecksumType {
w.Header().Set(xhttp.AmzChecksumType, v)
continue
}
cksum := NewChecksumString(k, v)
if cksum == nil {
continue
@@ -458,19 +535,11 @@ func GetContentChecksum(h http.Header) (*Checksum, error) {
var res *Checksum
for _, header := range trailing {
var duplicates bool
switch {
case strings.EqualFold(header, ChecksumCRC32C.Key()):
duplicates = res != nil
res = NewChecksumWithType(ChecksumCRC32C|ChecksumTrailing, "")
case strings.EqualFold(header, ChecksumCRC32.Key()):
duplicates = res != nil
res = NewChecksumWithType(ChecksumCRC32|ChecksumTrailing, "")
case strings.EqualFold(header, ChecksumSHA256.Key()):
duplicates = res != nil
res = NewChecksumWithType(ChecksumSHA256|ChecksumTrailing, "")
case strings.EqualFold(header, ChecksumSHA1.Key()):
duplicates = res != nil
res = NewChecksumWithType(ChecksumSHA1|ChecksumTrailing, "")
for _, t := range BaseChecksumTypes {
if strings.EqualFold(t.Key(), header) {
duplicates = res != nil
res = NewChecksumWithType(t|ChecksumTrailing, "")
}
}
if duplicates {
return nil, ErrInvalidChecksum
@@ -500,7 +569,13 @@ func getContentChecksum(h http.Header) (t ChecksumType, s string) {
t = ChecksumNone
alg := h.Get(xhttp.AmzChecksumAlgo)
if alg != "" {
t |= NewChecksumType(alg)
t |= NewChecksumHeader(h)
if h.Get(xhttp.AmzChecksumType) == xhttp.AmzChecksumTypeFullObject {
if !t.CanMerge() {
return ChecksumInvalid, ""
}
t |= ChecksumFullObject
}
if t.IsSet() {
hdr := t.Key()
if s = h.Get(hdr); s == "" {
@@ -519,12 +594,19 @@ func getContentChecksum(h http.Header) (t ChecksumType, s string) {
t = c
s = got
}
if h.Get(xhttp.AmzChecksumType) == xhttp.AmzChecksumTypeFullObject {
if !t.CanMerge() {
t = ChecksumInvalid
s = ""
return
}
t |= ChecksumFullObject
}
return
}
}
checkType(ChecksumCRC32)
checkType(ChecksumCRC32C)
checkType(ChecksumSHA1)
checkType(ChecksumSHA256)
for _, t := range BaseChecksumTypes {
checkType(t)
}
return t, s
}