support CRC32 Checksums on single drive setup (#15873)

cmd/erasure-multipart.go

@@ -1103,7 +1103,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 					PartNumber: part.PartNumber,
 				}
 			}
-			checksumCombined = append(checksumCombined, cs.Raw()...)
+			checksumCombined = append(checksumCombined, cs.Raw...)
 		}
 
 		// All parts except the last part has to be at least 5MB.

cmd/erasure-single-drive.go

@@ -1017,6 +1017,11 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 	}
 
 	fi.DataDir = mustGetUUID()
+	fi.Checksum = opts.WantChecksum.AppendTo(nil)
+	if opts.EncryptFn != nil {
+		fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
+	}
+
 	uniqueID := mustGetUUID()
 	tempObj := uniqueID
 
@@ -2593,6 +2598,7 @@ func (es *erasureSingle) ListObjectParts(ctx context.Context, bucket, object, up
 	result.MaxParts = maxParts
 	result.PartNumberMarker = partNumberMarker
 	result.UserDefined = cloneMSS(fi.Metadata)
+	result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum]
 
 	// For empty number of parts or maxParts as zero, return right here.
 	if len(fi.Parts) == 0 || maxParts == 0 {
@@ -2613,10 +2619,15 @@ func (es *erasureSingle) ListObjectParts(ctx context.Context, bucket, object, up
 	count := maxParts
 	for _, part := range parts {
 		result.Parts = append(result.Parts, PartInfo{
-			PartNumber:   part.Number,
-			ETag:         part.ETag,
-			LastModified: fi.ModTime,
-			Size:         part.Size,
+			PartNumber:     part.Number,
+			ETag:           part.ETag,
+			LastModified:   fi.ModTime,
+			ActualSize:     part.ActualSize,
+			Size:           part.Size,
+			ChecksumCRC32:  part.Checksums["CRC32"],
+			ChecksumCRC32C: part.Checksums["CRC32C"],
+			ChecksumSHA1:   part.Checksums["SHA1"],
+			ChecksumSHA256: part.Checksums["SHA256"],
 		})
 		count--
 		if count == 0 {
@@ -2685,6 +2696,21 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, err
 	}
 
+	// Checksum type set when upload started.
+	var checksumType hash.ChecksumType
+	if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
+		checksumType = hash.NewChecksumType(cs)
+		if opts.WantChecksum != nil && !opts.WantChecksum.Type.Is(checksumType) {
+			return oi, InvalidArgument{
+				Bucket: bucket,
+				Object: fi.Name,
+				Err:    fmt.Errorf("checksum type mismatch"),
+			}
+		}
+	}
+
+	var checksumCombined []byte
+
 	// However, in case of encryption, the persisted part ETags don't match
 	// what we have sent to the client during PutObjectPart. The reason is
 	// that ETags are encrypted. Hence, the client will send a list of complete
@@ -2731,6 +2757,7 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
 			}
 			return oi, invp
 		}
+		expPart := currentFI.Parts[partIdx]
 
 		// ensure that part ETag is canonicalized to strip off extraneous quotes
 		part.ETag = canonicalizeETag(part.ETag)
@@ -2744,27 +2771,58 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
 			return oi, invp
 		}
 
+		if checksumType.IsSet() {
+			crc := expPart.Checksums[checksumType.String()]
+			if crc == "" {
+				return oi, InvalidPart{
+					PartNumber: part.PartNumber,
+				}
+			}
+			wantCS := map[string]string{
+				hash.ChecksumCRC32.String():  part.ChecksumCRC32,
+				hash.ChecksumCRC32C.String(): part.ChecksumCRC32C,
+				hash.ChecksumSHA1.String():   part.ChecksumSHA1,
+				hash.ChecksumSHA256.String(): part.ChecksumSHA256,
+			}
+			if wantCS[checksumType.String()] != crc {
+				return oi, InvalidPart{
+					PartNumber: part.PartNumber,
+					ExpETag:    wantCS[checksumType.String()],
+					GotETag:    crc,
+				}
+			}
+			cs := hash.NewChecksumString(checksumType.String(), crc)
+			if !cs.Valid() {
+				return oi, InvalidPart{
+					PartNumber: part.PartNumber,
+				}
+			}
+			checksumCombined = append(checksumCombined, cs.Raw...)
+		}
+
 		// All parts except the last part has to be atleast 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(currentFI.Parts[partIdx].ActualSize) {
 			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
-				PartSize:   currentFI.Parts[partIdx].ActualSize,
+				PartSize:   expPart.ActualSize,
 				PartETag:   part.ETag,
 			}
 		}
 
 		// Save for total object size.
-		objectSize += currentFI.Parts[partIdx].Size
+		objectSize += expPart.Size
 
 		// Save the consolidated actual size.
-		objectActualSize += currentFI.Parts[partIdx].ActualSize
+		objectActualSize += expPart.ActualSize
 
 		// Add incoming parts.
 		fi.Parts[i] = ObjectPartInfo{
 			Number:     part.PartNumber,
-			Size:       currentFI.Parts[partIdx].Size,
-			ActualSize: currentFI.Parts[partIdx].ActualSize,
-			Index:      currentFI.Parts[partIdx].Index,
+			Size:       expPart.Size,
+			ActualSize: expPart.ActualSize,
+			ModTime:    expPart.ModTime,
+			Index:      expPart.Index,
+			Checksums:  nil, // Not transferred since we do not need it.
 		}
 	}
 

cmd/object-handlers.go

@@ -1797,7 +1797,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		if err = actualReader.AddChecksum(r, false); err != nil {
+		if err = actualReader.AddChecksum(r, globalIsGateway); err != nil {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
 			return
 		}
@@ -1818,7 +1818,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if err := hashReader.AddChecksum(r, size < 0); err != nil {
+	if err := hashReader.AddChecksum(r, size < 0 || globalIsGateway); err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
 		return
 	}
@@ -2129,7 +2129,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if err = hreader.AddChecksum(r, false); err != nil {
+	if err = hreader.AddChecksum(r, globalIsGateway); err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
 		return
 	}

cmd/object-multipart-handlers.go

@@ -390,7 +390,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		if err = actualReader.AddChecksum(r, false); err != nil {
+		if err = actualReader.AddChecksum(r, globalIsGateway); err != nil {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
 			return
 		}
@@ -411,7 +411,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if err := hashReader.AddChecksum(r, size < 0); err != nil {
+	if err := hashReader.AddChecksum(r, size < 0 || globalIsGateway); err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
 		return
 	}

internal/hash/checksum.go

@@ -18,9 +18,11 @@
 package hash
 
 import (
+	"bytes"
 	"crypto/sha1"
 	"encoding/base64"
 	"encoding/binary"
+	"fmt"
 	"hash"
 	"hash/crc32"
 	"net/http"
@@ -61,6 +63,7 @@ const (
 type Checksum struct {
 	Type    ChecksumType
 	Encoded string
+	Raw     []byte
 }
 
 // Is returns if c is all of t.
@@ -169,7 +172,8 @@ func NewChecksumFromData(t ChecksumType, data []byte) *Checksum {
 	}
 	h := t.Hasher()
 	h.Write(data)
-	c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(h.Sum(nil))}
+	raw := h.Sum(nil)
+	c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(raw), Raw: raw}
 	if !c.Valid() {
 		return nil
 	}
@@ -200,19 +204,27 @@ func ReadCheckSums(b []byte) map[string]string {
 	return res
 }
 
-// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value.
-func NewChecksumString(alg, value string) *Checksum {
-	t := NewChecksumType(alg)
-	if !t.IsSet() {
+// NewChecksumWithType is similar to NewChecksumString but expects input algo of ChecksumType.
+func NewChecksumWithType(alg ChecksumType, value string) *Checksum {
+	if !alg.IsSet() {
 		return nil
 	}
-	c := Checksum{Type: t, Encoded: value}
+	bvalue, err := base64.StdEncoding.DecodeString(value)
+	if err != nil {
+		return nil
+	}
+	c := Checksum{Type: alg, Encoded: value, Raw: bvalue}
 	if !c.Valid() {
 		return nil
 	}
 	return &c
 }
 
+// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value.
+func NewChecksumString(alg, value string) *Checksum {
+	return NewChecksumWithType(NewChecksumType(alg), value)
+}
+
 // AppendTo will append the checksum to b.
 // ReadCheckSums reads the values back.
 func (c *Checksum) AppendTo(b []byte) []byte {
@@ -221,7 +233,7 @@ func (c *Checksum) AppendTo(b []byte) []byte {
 	}
 	var tmp [binary.MaxVarintLen32]byte
 	n := binary.PutUvarint(tmp[:], uint64(c.Type))
-	crc := c.Raw()
+	crc := c.Raw
 	if len(crc) != c.Type.RawByteLen() {
 		return b
 	}
@@ -238,19 +250,10 @@ func (c Checksum) Valid() bool {
 	if len(c.Encoded) == 0 || c.Type.Is(ChecksumTrailing) {
 		return c.Type.Is(ChecksumNone) || c.Type.Is(ChecksumTrailing)
 	}
-	raw := c.Raw()
+	raw := c.Raw
 	return c.Type.RawByteLen() == len(raw)
 }
 
-// Raw returns the Raw checksum.
-func (c Checksum) Raw() []byte {
-	if len(c.Encoded) == 0 {
-		return nil
-	}
-	v, _ := base64.StdEncoding.DecodeString(c.Encoded)
-	return v
-}
-
 // Matches returns whether given content matches c.
 func (c Checksum) Matches(content []byte) error {
 	if len(c.Encoded) == 0 {
@@ -261,11 +264,11 @@ func (c Checksum) Matches(content []byte) error {
 	if err != nil {
 		return err
 	}
-	got := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-	if got != c.Encoded {
+	sum := hasher.Sum(nil)
+	if !bytes.Equal(sum, c.Raw) {
 		return ChecksumMismatch{
 			Want: c.Encoded,
-			Got:  got,
+			Got:  base64.StdEncoding.EncodeToString(sum),
 		}
 	}
 	return nil
@@ -292,13 +295,13 @@ func TransferChecksumHeader(w http.ResponseWriter, r *http.Request) {
 // AddChecksumHeader will transfer any checksum value that has been checked.
 func AddChecksumHeader(w http.ResponseWriter, c map[string]string) {
 	for k, v := range c {
-		typ := NewChecksumType(k)
-		if !typ.IsSet() {
+		fmt.Println(c, v)
+		cksum := NewChecksumString(k, v)
+		if cksum == nil {
 			continue
 		}
-		crc := Checksum{Type: typ, Encoded: v}
-		if crc.Valid() {
-			w.Header().Set(typ.Key(), v)
+		if cksum.Valid() {
+			w.Header().Set(cksum.Type.Key(), v)
 		}
 	}
 }
@@ -314,12 +317,11 @@ func GetContentChecksum(r *http.Request) (*Checksum, error) {
 		}
 		return nil, ErrInvalidChecksum
 	}
-	c := Checksum{Type: t, Encoded: s}
-	if !c.Valid() {
+	cksum := NewChecksumWithType(t, s)
+	if cksum == nil {
 		return nil, ErrInvalidChecksum
 	}
-
-	return &c, nil
+	return cksum, nil
 }
 
 // getContentChecksum returns content checksum type and value.

internal/hash/reader.go

@@ -186,7 +186,7 @@ func (r *Reader) Read(p []byte) (int, error) {
 			}
 		}
 		if r.contentHasher != nil {
-			if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw(), sum) {
+			if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw, sum) {
 				err := ChecksumMismatch{
 					Want: r.contentHash.Encoded,
 					Got:  base64.StdEncoding.EncodeToString(sum),