Signed trailers for signature v4 (#16484)

internal/hash/checksum.go

@@ -303,8 +303,8 @@ func (c Checksum) Valid() bool {
 	if c.Type == ChecksumInvalid {
 		return false
 	}
-	if len(c.Encoded) == 0 || c.Type.Is(ChecksumTrailing) {
-		return c.Type.Is(ChecksumNone) || c.Type.Is(ChecksumTrailing)
+	if len(c.Encoded) == 0 || c.Type.Trailing() {
+		return c.Type.Is(ChecksumNone) || c.Type.Trailing()
 	}
 	raw := c.Raw
 	return c.Type.RawByteLen() == len(raw)
@@ -339,10 +339,21 @@ func (c *Checksum) AsMap() map[string]string {
 }
 
 // TransferChecksumHeader will transfer any checksum value that has been checked.
+// If checksum was trailing, they must have been added to r.Trailer.
 func TransferChecksumHeader(w http.ResponseWriter, r *http.Request) {
-	t, s := getContentChecksum(r)
-	if !t.IsSet() || t.Is(ChecksumTrailing) {
-		// TODO: Add trailing when we can read it.
+	c, err := GetContentChecksum(r)
+	if err != nil || c == nil {
+		return
+	}
+	t, s := c.Type, c.Encoded
+	if !c.Type.IsSet() {
+		return
+	}
+	if c.Type.Is(ChecksumTrailing) {
+		val := r.Trailer.Get(t.Key())
+		if val != "" {
+			w.Header().Set(t.Key(), val)
+		}
 		return
 	}
 	w.Header().Set(t.Key(), s)
@@ -365,6 +376,32 @@ func AddChecksumHeader(w http.ResponseWriter, c map[string]string) {
 // Returns ErrInvalidChecksum if so.
 // Returns nil, nil if no checksum.
 func GetContentChecksum(r *http.Request) (*Checksum, error) {
+	if trailing := r.Header.Values(xhttp.AmzTrailer); len(trailing) > 0 {
+		var res *Checksum
+		for _, header := range trailing {
+			var duplicates bool
+			switch {
+			case strings.EqualFold(header, ChecksumCRC32C.Key()):
+				duplicates = res != nil
+				res = NewChecksumWithType(ChecksumCRC32C|ChecksumTrailing, "")
+			case strings.EqualFold(header, ChecksumCRC32.Key()):
+				duplicates = res != nil
+				res = NewChecksumWithType(ChecksumCRC32|ChecksumTrailing, "")
+			case strings.EqualFold(header, ChecksumSHA256.Key()):
+				duplicates = res != nil
+				res = NewChecksumWithType(ChecksumSHA256|ChecksumTrailing, "")
+			case strings.EqualFold(header, ChecksumSHA1.Key()):
+				duplicates = res != nil
+				res = NewChecksumWithType(ChecksumSHA1|ChecksumTrailing, "")
+			}
+			if duplicates {
+				return nil, ErrInvalidChecksum
+			}
+		}
+		if res != nil {
+			return res, nil
+		}
+	}
 	t, s := getContentChecksum(r)
 	if t == ChecksumNone {
 		if s == "" {
@@ -389,11 +426,6 @@ func getContentChecksum(r *http.Request) (t ChecksumType, s string) {
 		if t.IsSet() {
 			hdr := t.Key()
 			if s = r.Header.Get(hdr); s == "" {
-				if strings.EqualFold(r.Header.Get(xhttp.AmzTrailer), hdr) {
-					t |= ChecksumTrailing
-				} else {
-					t = ChecksumInvalid
-				}
 				return ChecksumNone, ""
 			}
 		}
@@ -409,6 +441,7 @@ func getContentChecksum(r *http.Request) (t ChecksumType, s string) {
 				t = c
 				s = got
 			}
+			return
 		}
 	}
 	checkType(ChecksumCRC32)

internal/hash/reader.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/hash/sha256"
+	"github.com/minio/minio/internal/ioutil"
 )
 
 // A Reader wraps an io.Reader and computes the MD5 checksum
@@ -51,6 +52,8 @@ type Reader struct {
 	contentHash   Checksum
 	contentHasher hash.Hash
 
+	trailer http.Header
+
 	sha256 hash.Hash
 }
 
@@ -107,7 +110,7 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
 		r.checksum = MD5
 		r.contentSHA256 = SHA256
 		if r.size < 0 && size >= 0 {
-			r.src = etag.Wrap(io.LimitReader(r.src, size), r.src)
+			r.src = etag.Wrap(ioutil.HardLimitReader(r.src, size), r.src)
 			r.size = size
 		}
 		if r.actualSize <= 0 && actualSize >= 0 {
@@ -117,7 +120,7 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
 	}
 
 	if size >= 0 {
-		r := io.LimitReader(src, size)
+		r := ioutil.HardLimitReader(src, size)
 		if _, ok := src.(etag.Tagger); !ok {
 			src = etag.NewReader(r, MD5)
 		} else {
@@ -155,10 +158,14 @@ func (r *Reader) AddChecksum(req *http.Request, ignoreValue bool) error {
 		return nil
 	}
 	r.contentHash = *cs
-	if cs.Type.Trailing() || ignoreValue {
-		// Ignore until we have trailing headers.
+	if cs.Type.Trailing() {
+		r.trailer = req.Trailer
+	}
+	if ignoreValue {
+		// Do not validate, but allow for transfer
 		return nil
 	}
+
 	r.contentHasher = cs.Type.Hasher()
 	if r.contentHasher == nil {
 		return ErrInvalidChecksum
@@ -186,6 +193,14 @@ func (r *Reader) Read(p []byte) (int, error) {
 			}
 		}
 		if r.contentHasher != nil {
+			if r.contentHash.Type.Trailing() {
+				var err error
+				r.contentHash.Encoded = r.trailer.Get(r.contentHash.Type.Key())
+				r.contentHash.Raw, err = base64.StdEncoding.DecodeString(r.contentHash.Encoded)
+				if err != nil || len(r.contentHash.Raw) == 0 {
+					return 0, ChecksumMismatch{Got: r.contentHash.Encoded}
+				}
+			}
 			if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw, sum) {
 				err := ChecksumMismatch{
 					Want: r.contentHash.Encoded,
@@ -276,6 +291,9 @@ func (r *Reader) ContentCRC() map[string]string {
 	if r.contentHash.Type == ChecksumNone || !r.contentHash.Valid() {
 		return nil
 	}
+	if r.contentHash.Type.Trailing() {
+		return map[string]string{r.contentHash.Type.String(): r.trailer.Get(r.contentHash.Type.Key())}
+	}
 	return map[string]string{r.contentHash.Type.String(): r.contentHash.Encoded}
 }
 

internal/hash/reader_test.go

@@ -23,6 +23,8 @@ import (
 	"fmt"
 	"io"
 	"testing"
+
+	"github.com/minio/minio/internal/ioutil"
 )
 
 // Tests functions like Size(), MD5*(), SHA256*()
@@ -79,7 +81,7 @@ func TestHashReaderVerification(t *testing.T) {
 		md5hex, sha256hex string
 		err               error
 	}{
-		{
+		0: {
 			desc:       "Success, no checksum verification provided.",
 			src:        bytes.NewReader([]byte("abcd")),
 			size:       4,
@@ -124,7 +126,7 @@ func TestHashReaderVerification(t *testing.T) {
 				CalculatedSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
 			},
 		},
-		{
+		5: {
 			desc:       "Correct sha256, nested",
 			src:        mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
 			size:       4,
@@ -137,13 +139,15 @@ func TestHashReaderVerification(t *testing.T) {
 			size:       4,
 			actualSize: -1,
 			sha256hex:  "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
+			err:        ioutil.ErrOverread,
 		},
-		{
+		7: {
 			desc:       "Correct sha256, nested, truncated, swapped",
 			src:        mustReader(t, bytes.NewReader([]byte("abcd-more-stuff-to-be ignored")), 4, "", "", -1),
 			size:       4,
 			actualSize: -1,
 			sha256hex:  "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
+			err:        ioutil.ErrOverread,
 		},
 		{
 			desc:       "Incorrect MD5, nested",
@@ -162,6 +166,7 @@ func TestHashReaderVerification(t *testing.T) {
 			size:       4,
 			actualSize: 4,
 			sha256hex:  "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
+			err:        ioutil.ErrOverread,
 		},
 		{
 			desc:       "Correct MD5, nested",
@@ -177,6 +182,7 @@ func TestHashReaderVerification(t *testing.T) {
 			actualSize: 4,
 			sha256hex:  "",
 			md5hex:     "e2fc714c4727ee9395f324cd2e7f331f",
+			err:        ioutil.ErrOverread,
 		},
 		{
 			desc:       "Correct MD5, nested, truncated",
@@ -184,6 +190,7 @@ func TestHashReaderVerification(t *testing.T) {
 			size:       4,
 			actualSize: 4,
 			md5hex:     "e2fc714c4727ee9395f324cd2e7f331f",
+			err:        ioutil.ErrOverread,
 		},
 	}
 	for i, testCase := range testCases {
@@ -194,6 +201,10 @@ func TestHashReaderVerification(t *testing.T) {
 			}
 			_, err = io.Copy(io.Discard, r)
 			if err != nil {
+				if testCase.err == nil {
+					t.Errorf("Test %q; got unexpected error: %v", testCase.desc, err)
+					return
+				}
 				if err.Error() != testCase.err.Error() {
 					t.Errorf("Test %q: Expected error %s, got error %s", testCase.desc, testCase.err, err)
 				}