Add extended checksum support (#15433)

internal/hash/checksum.go

@@ -0,0 +1,359 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package hash
+
+import (
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/binary"
+	"hash"
+	"hash/crc32"
+	"net/http"
+	"strings"
+
+	"github.com/minio/minio/internal/hash/sha256"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// MinIOMultipartChecksum is as metadata on multipart uploads to indicate checksum type.
+const MinIOMultipartChecksum = "x-minio-multipart-checksum"
+
+// ChecksumType contains information about the checksum type.
+type ChecksumType uint32
+
+const (
+
+	// ChecksumTrailing indicates the checksum will be sent in the trailing header.
+	// Another checksum type will be set.
+	ChecksumTrailing ChecksumType = 1 << iota
+
+	// ChecksumSHA256 indicates a SHA256 checksum.
+	ChecksumSHA256
+	// ChecksumSHA1 indicates a SHA-1 checksum.
+	ChecksumSHA1
+	// ChecksumCRC32 indicates a CRC32 checksum with IEEE table.
+	ChecksumCRC32
+	// ChecksumCRC32C indicates a CRC32 checksum with Castagnoli table.
+	ChecksumCRC32C
+	// ChecksumInvalid indicates an invalid checksum.
+	ChecksumInvalid
+
+	// ChecksumNone indicates no checksum.
+	ChecksumNone ChecksumType = 0
+)
+
+// Checksum is a type and base 64 encoded value.
+type Checksum struct {
+	Type    ChecksumType
+	Encoded string
+}
+
+// Is returns if c is all of t.
+func (c ChecksumType) Is(t ChecksumType) bool {
+	if t == ChecksumNone {
+		return c == ChecksumNone
+	}
+	return c&t == t
+}
+
+// Key returns the header key.
+// returns empty string if invalid or none.
+func (c ChecksumType) Key() string {
+	switch {
+	case c.Is(ChecksumCRC32):
+		return xhttp.AmzChecksumCRC32
+	case c.Is(ChecksumCRC32C):
+		return xhttp.AmzChecksumCRC32C
+	case c.Is(ChecksumSHA1):
+		return xhttp.AmzChecksumSHA1
+	case c.Is(ChecksumSHA256):
+		return xhttp.AmzChecksumSHA256
+	}
+	return ""
+}
+
+// RawByteLen returns the size of the un-encoded checksum.
+func (c ChecksumType) RawByteLen() int {
+	switch {
+	case c.Is(ChecksumCRC32):
+		return 4
+	case c.Is(ChecksumCRC32C):
+		return 4
+	case c.Is(ChecksumSHA1):
+		return sha1.Size
+	case c.Is(ChecksumSHA256):
+		return sha256.Size
+	}
+	return 0
+}
+
+// IsSet returns whether the type is valid and known.
+func (c ChecksumType) IsSet() bool {
+	return !c.Is(ChecksumInvalid) && !c.Is(ChecksumNone)
+}
+
+// NewChecksumType returns a checksum type based on the algorithm string.
+func NewChecksumType(alg string) ChecksumType {
+	switch strings.ToUpper(alg) {
+	case "CRC32":
+		return ChecksumCRC32
+	case "CRC32C":
+		return ChecksumCRC32C
+	case "SHA1":
+		return ChecksumSHA1
+	case "SHA256":
+		return ChecksumSHA256
+	case "":
+		return ChecksumNone
+	}
+	return ChecksumInvalid
+}
+
+// String returns the type as a string.
+func (c ChecksumType) String() string {
+	switch {
+	case c.Is(ChecksumCRC32):
+		return "CRC32"
+	case c.Is(ChecksumCRC32C):
+		return "CRC32C"
+	case c.Is(ChecksumSHA1):
+		return "SHA1"
+	case c.Is(ChecksumSHA256):
+		return "SHA256"
+	case c.Is(ChecksumNone):
+		return ""
+	}
+	return "invalid"
+}
+
+// Hasher returns a hasher corresponding to the checksum type.
+// Returns nil if no checksum.
+func (c ChecksumType) Hasher() hash.Hash {
+	switch {
+	case c.Is(ChecksumCRC32):
+		return crc32.NewIEEE()
+	case c.Is(ChecksumCRC32C):
+		return crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case c.Is(ChecksumSHA1):
+		return sha1.New()
+	case c.Is(ChecksumSHA256):
+		return sha256.New()
+	}
+	return nil
+}
+
+// Trailing return whether the checksum is traling.
+func (c ChecksumType) Trailing() bool {
+	return c.Is(ChecksumTrailing)
+}
+
+// NewChecksumFromData returns a new checksum from specified algorithm and base64 encoded value.
+func NewChecksumFromData(t ChecksumType, data []byte) *Checksum {
+	if !t.IsSet() {
+		return nil
+	}
+	h := t.Hasher()
+	h.Write(data)
+	c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(h.Sum(nil))}
+	if !c.Valid() {
+		return nil
+	}
+	return &c
+}
+
+// ReadCheckSums will read checksums from b and return them.
+func ReadCheckSums(b []byte) map[string]string {
+	res := make(map[string]string, 1)
+	for len(b) > 0 {
+		t, n := binary.Uvarint(b)
+		if n < 0 {
+			break
+		}
+		b = b[n:]
+
+		typ := ChecksumType(t)
+		length := typ.RawByteLen()
+		if length == 0 || len(b) < length {
+			break
+		}
+		res[typ.String()] = base64.StdEncoding.EncodeToString(b[:length])
+		b = b[length:]
+	}
+	if len(res) == 0 {
+		res = nil
+	}
+	return res
+}
+
+// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value.
+func NewChecksumString(alg, value string) *Checksum {
+	t := NewChecksumType(alg)
+	if !t.IsSet() {
+		return nil
+	}
+	c := Checksum{Type: t, Encoded: value}
+	if !c.Valid() {
+		return nil
+	}
+	return &c
+}
+
+// AppendTo will append the checksum to b.
+// ReadCheckSums reads the values back.
+func (c Checksum) AppendTo(b []byte) []byte {
+	var tmp [binary.MaxVarintLen32]byte
+	n := binary.PutUvarint(tmp[:], uint64(c.Type))
+	crc := c.Raw()
+	if len(crc) != c.Type.RawByteLen() {
+		return b
+	}
+	b = append(b, tmp[:n]...)
+	b = append(b, crc...)
+	return b
+}
+
+// Valid returns whether checksum is valid.
+func (c Checksum) Valid() bool {
+	if c.Type == ChecksumInvalid {
+		return false
+	}
+	if len(c.Encoded) == 0 || c.Type.Is(ChecksumTrailing) {
+		return c.Type.Is(ChecksumNone) || c.Type.Is(ChecksumTrailing)
+	}
+	raw := c.Raw()
+	return c.Type.RawByteLen() == len(raw)
+}
+
+// Raw returns the Raw checksum.
+func (c Checksum) Raw() []byte {
+	if len(c.Encoded) == 0 {
+		return nil
+	}
+	v, _ := base64.StdEncoding.DecodeString(c.Encoded)
+	return v
+}
+
+// Matches returns whether given content matches c.
+func (c Checksum) Matches(content []byte) error {
+	if len(c.Encoded) == 0 {
+		return nil
+	}
+	hasher := c.Type.Hasher()
+	_, err := hasher.Write(content)
+	if err != nil {
+		return err
+	}
+	got := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	if got != c.Encoded {
+		return ChecksumMismatch{
+			Want: c.Encoded,
+			Got:  got,
+		}
+	}
+	return nil
+}
+
+// AsMap returns the
+func (c *Checksum) AsMap() map[string]string {
+	if c == nil || !c.Valid() {
+		return nil
+	}
+	return map[string]string{c.Type.String(): c.Encoded}
+}
+
+// TransferChecksumHeader will transfer any checksum value that has been checked.
+func TransferChecksumHeader(w http.ResponseWriter, r *http.Request) {
+	t, s := getContentChecksum(r)
+	if !t.IsSet() || t.Is(ChecksumTrailing) {
+		// TODO: Add trailing when we can read it.
+		return
+	}
+	w.Header().Set(t.Key(), s)
+}
+
+// AddChecksumHeader will transfer any checksum value that has been checked.
+func AddChecksumHeader(w http.ResponseWriter, c map[string]string) {
+	for k, v := range c {
+		typ := NewChecksumType(k)
+		if !typ.IsSet() {
+			continue
+		}
+		crc := Checksum{Type: typ, Encoded: v}
+		if crc.Valid() {
+			w.Header().Set(typ.Key(), v)
+		}
+	}
+}
+
+// GetContentChecksum returns content checksum.
+// Returns ErrInvalidChecksum if so.
+// Returns nil, nil if no checksum.
+func GetContentChecksum(r *http.Request) (*Checksum, error) {
+	t, s := getContentChecksum(r)
+	if t == ChecksumNone {
+		if s == "" {
+			return nil, nil
+		}
+		return nil, ErrInvalidChecksum
+	}
+	c := Checksum{Type: t, Encoded: s}
+	if !c.Valid() {
+		return nil, ErrInvalidChecksum
+	}
+
+	return &c, nil
+}
+
+// getContentChecksum returns content checksum type and value.
+// Returns ChecksumInvalid if so.
+func getContentChecksum(r *http.Request) (t ChecksumType, s string) {
+	t = ChecksumNone
+	alg := r.Header.Get(xhttp.AmzChecksumAlgo)
+	if alg != "" {
+		t |= NewChecksumType(alg)
+		if t.IsSet() {
+			hdr := t.Key()
+			if s = r.Header.Get(hdr); s == "" {
+				if strings.EqualFold(r.Header.Get(xhttp.AmzTrailer), hdr) {
+					t |= ChecksumTrailing
+				} else {
+					t = ChecksumInvalid
+				}
+				return ChecksumNone, ""
+			}
+		}
+		return t, s
+	}
+	checkType := func(c ChecksumType) {
+		if got := r.Header.Get(c.Key()); got != "" {
+			// If already set, invalid
+			if t != ChecksumNone {
+				t = ChecksumInvalid
+				s = ""
+			} else {
+				t = c
+				s = got
+			}
+		}
+	}
+	checkType(ChecksumCRC32)
+	checkType(ChecksumCRC32C)
+	checkType(ChecksumSHA1)
+	checkType(ChecksumSHA256)
+	return t, s
+}

internal/hash/errors.go

@@ -48,3 +48,13 @@ type ErrSizeMismatch struct {
 func (e ErrSizeMismatch) Error() string {
 	return fmt.Sprintf("Size mismatch: got %d, want %d", e.Got, e.Want)
 }
+
+// ChecksumMismatch - when content checksum does not match with what was sent from client.
+type ChecksumMismatch struct {
+	Want string
+	Got  string
+}
+
+func (e ChecksumMismatch) Error() string {
+	return "Bad checksum: Want " + e.Want + " does not match calculated " + e.Got
+}

internal/hash/reader.go

@@ -24,6 +24,7 @@ import (
 	"errors"
 	"hash"
 	"io"
+	"net/http"
 
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/hash/sha256"
@@ -46,6 +47,10 @@ type Reader struct {
 	checksum      etag.ETag
 	contentSHA256 []byte
 
+	// Content checksum
+	contentHash   Checksum
+	contentHasher hash.Hash
+
 	sha256 hash.Hash
 }
 
@@ -83,7 +88,7 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
 		if r.bytesRead > 0 {
 			return nil, errors.New("hash: already read from hash reader")
 		}
-		if len(r.checksum) != 0 && len(MD5) != 0 && !etag.Equal(r.checksum, etag.ETag(MD5)) {
+		if len(r.checksum) != 0 && len(MD5) != 0 && !etag.Equal(r.checksum, MD5) {
 			return nil, BadDigest{
 				ExpectedMD5:   r.checksum.String(),
 				CalculatedMD5: md5Hex,
@@ -99,7 +104,7 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
 			return nil, ErrSizeMismatch{Want: r.size, Got: size}
 		}
 
-		r.checksum = etag.ETag(MD5)
+		r.checksum = MD5
 		r.contentSHA256 = SHA256
 		if r.size < 0 && size >= 0 {
 			r.src = etag.Wrap(io.LimitReader(r.src, size), r.src)
@@ -114,33 +119,62 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
 	if size >= 0 {
 		r := io.LimitReader(src, size)
 		if _, ok := src.(etag.Tagger); !ok {
-			src = etag.NewReader(r, etag.ETag(MD5))
+			src = etag.NewReader(r, MD5)
 		} else {
 			src = etag.Wrap(r, src)
 		}
 	} else if _, ok := src.(etag.Tagger); !ok {
-		src = etag.NewReader(src, etag.ETag(MD5))
+		src = etag.NewReader(src, MD5)
 	}
-	var hash hash.Hash
+	var h hash.Hash
 	if len(SHA256) != 0 {
-		hash = sha256.New()
+		h = sha256.New()
 	}
 	return &Reader{
 		src:           src,
 		size:          size,
 		actualSize:    actualSize,
-		checksum:      etag.ETag(MD5),
+		checksum:      MD5,
 		contentSHA256: SHA256,
-		sha256:        hash,
+		sha256:        h,
 	}, nil
 }
 
+// ErrInvalidChecksum is returned when an invalid checksum is provided in headers.
+var ErrInvalidChecksum = errors.New("invalid checksum")
+
+// AddChecksum will add checksum checks as specified in
+// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+// Returns ErrInvalidChecksum if a problem with the checksum is found.
+func (r *Reader) AddChecksum(req *http.Request, ignoreValue bool) error {
+	cs, err := GetContentChecksum(req)
+	if err != nil {
+		return ErrInvalidChecksum
+	}
+	if cs == nil {
+		return nil
+	}
+	r.contentHash = *cs
+	if cs.Type.Trailing() || ignoreValue {
+		// Ignore until we have trailing headers.
+		return nil
+	}
+	r.contentHasher = cs.Type.Hasher()
+	if r.contentHasher == nil {
+		return ErrInvalidChecksum
+	}
+	return nil
+}
+
 func (r *Reader) Read(p []byte) (int, error) {
 	n, err := r.src.Read(p)
 	r.bytesRead += int64(n)
 	if r.sha256 != nil {
 		r.sha256.Write(p[:n])
 	}
+	if r.contentHasher != nil {
+		r.contentHasher.Write(p[:n])
+	}
 
 	if err == io.EOF { // Verify content SHA256, if set.
 		if r.sha256 != nil {
@@ -151,6 +185,15 @@ func (r *Reader) Read(p []byte) (int, error) {
 				}
 			}
 		}
+		if r.contentHasher != nil {
+			if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw(), sum) {
+				err := ChecksumMismatch{
+					Want: r.contentHash.Encoded,
+					Got:  base64.StdEncoding.EncodeToString(sum),
+				}
+				return n, err
+			}
+		}
 	}
 	if err != nil && err != io.EOF {
 		if v, ok := err.(etag.VerifyError); ok {
@@ -223,6 +266,19 @@ func (r *Reader) SHA256HexString() string {
 	return hex.EncodeToString(r.contentSHA256)
 }
 
+// ContentCRCType returns the content checksum type.
+func (r *Reader) ContentCRCType() ChecksumType {
+	return r.contentHash.Type
+}
+
+// ContentCRC returns the content crc if set.
+func (r *Reader) ContentCRC() map[string]string {
+	if r.contentHash.Type == ChecksumNone || !r.contentHash.Valid() {
+		return nil
+	}
+	return map[string]string{r.contentHash.Type.String(): r.contentHash.Encoded}
+}
+
 var _ io.Closer = (*Reader)(nil) // compiler check
 
 // Close and release resources.

internal/hash/sha256/sh256_fips.go

@@ -33,3 +33,6 @@ func New() hash.Hash { return fipssha256.New() }
 
 // Sum256 returns the SHA256 checksum of the data.
 func Sum256(data []byte) [fipssha256.Size]byte { return fipssha256.Sum256(data) }
+
+// Size is the size of a SHA256 checksum in bytes.
+const Size = fipssha256.Size

internal/hash/sha256/sh256_nofips.go

@@ -32,3 +32,6 @@ func New() hash.Hash { return nofipssha256.New() }
 
 // Sum256 returns the SHA256 checksum of the data.
 func Sum256(data []byte) [nofipssha256.Size]byte { return nofipssha256.Sum256(data) }
+
+// Size is the size of a SHA256 checksum in bytes.
+const Size = nofipssha256.Size