Signed trailers for signature v4 (#16484)

cmd/streaming-signature-v4.go

@@ -24,9 +24,11 @@ import (
 	"bytes"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"hash"
 	"io"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -37,24 +39,53 @@ import (
 
 // Streaming AWS Signature Version '4' constants.
 const (
-	emptySHA256              = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-	streamingContentSHA256   = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
-	signV4ChunkedAlgorithm   = "AWS4-HMAC-SHA256-PAYLOAD"
-	streamingContentEncoding = "aws-chunked"
+	emptySHA256                   = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	streamingContentSHA256        = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	streamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
+	signV4ChunkedAlgorithm        = "AWS4-HMAC-SHA256-PAYLOAD"
+	signV4ChunkedAlgorithmTrailer = "AWS4-HMAC-SHA256-TRAILER"
+	streamingContentEncoding      = "aws-chunked"
+	awsTrailerHeader              = "X-Amz-Trailer"
+	trailerKVSeparator            = ":"
 )
 
 // getChunkSignature - get chunk signature.
-func getChunkSignature(cred auth.Credentials, seedSignature string, region string, date time.Time, hashedChunk string) string {
+// Does not update anything in cr.
+func (cr *s3ChunkedReader) getChunkSignature() string {
+	hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))
+
 	// Calculate string to sign.
-	stringToSign := signV4ChunkedAlgorithm + "\n" +
-		date.Format(iso8601Format) + "\n" +
-		getScope(date, region) + "\n" +
-		seedSignature + "\n" +
+	alg := signV4ChunkedAlgorithm + "\n"
+	stringToSign := alg +
+		cr.seedDate.Format(iso8601Format) + "\n" +
+		getScope(cr.seedDate, cr.region) + "\n" +
+		cr.seedSignature + "\n" +
 		emptySHA256 + "\n" +
 		hashedChunk
 
 	// Get hmac signing key.
-	signingKey := getSigningKey(cred.SecretKey, date, region, serviceS3)
+	signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate, cr.region, serviceS3)
+
+	// Calculate signature.
+	newSignature := getSignature(signingKey, stringToSign)
+
+	return newSignature
+}
+
+// getTrailerChunkSignature - get trailer chunk signature.
+func (cr *s3ChunkedReader) getTrailerChunkSignature() string {
+	hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))
+
+	// Calculate string to sign.
+	alg := signV4ChunkedAlgorithmTrailer + "\n"
+	stringToSign := alg +
+		cr.seedDate.Format(iso8601Format) + "\n" +
+		getScope(cr.seedDate, cr.region) + "\n" +
+		cr.seedSignature + "\n" +
+		hashedChunk
+
+	// Get hmac signing key.
+	signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate, cr.region, serviceS3)
 
 	// Calculate signature.
 	newSignature := getSignature(signingKey, stringToSign)
@@ -67,7 +98,7 @@ func getChunkSignature(cred auth.Credentials, seedSignature string, region strin
 //
 // returns signature, error otherwise if the signature mismatches or any other
 // error while parsing and validating.
-func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature string, region string, date time.Time, errCode APIErrorCode) {
+func calculateSeedSignature(r *http.Request, trailers bool) (cred auth.Credentials, signature string, region string, date time.Time, errCode APIErrorCode) {
 	// Copy request.
 	req := *r
 
@@ -82,6 +113,9 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 
 	// Payload streaming.
 	payload := streamingContentSHA256
+	if trailers {
+		payload = streamingContentSHA256Trailer
+	}
 
 	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
 	if payload != req.Header.Get(xhttp.AmzContentSha256) {
@@ -158,13 +192,24 @@ var errChunkTooBig = errors.New("chunk too big: choose chunk size <= 16MiB")
 //
 // NewChunkedReader is not needed by normal applications. The http package
 // automatically decodes chunking when reading response bodies.
-func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, APIErrorCode) {
-	cred, seedSignature, region, seedDate, errCode := calculateSeedSignature(req)
+func newSignV4ChunkedReader(req *http.Request, trailer bool) (io.ReadCloser, APIErrorCode) {
+	cred, seedSignature, region, seedDate, errCode := calculateSeedSignature(req, trailer)
 	if errCode != ErrNone {
 		return nil, errCode
 	}
 
+	if trailer {
+		// Discard anything unsigned.
+		req.Trailer = make(http.Header)
+		trailers := req.Header.Values(awsTrailerHeader)
+		for _, key := range trailers {
+			req.Trailer.Add(key, "")
+		}
+	} else {
+		req.Trailer = nil
+	}
 	return &s3ChunkedReader{
+		trailers:          req.Trailer,
 		reader:            bufio.NewReader(req.Body),
 		cred:              cred,
 		seedSignature:     seedSignature,
@@ -172,6 +217,7 @@ func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, APIErrorCode) {
 		region:            region,
 		chunkSHA256Writer: sha256.New(),
 		buffer:            make([]byte, 64*1024),
+		debug:             false,
 	}, ErrNone
 }
 
@@ -183,11 +229,13 @@ type s3ChunkedReader struct {
 	seedSignature string
 	seedDate      time.Time
 	region        string
+	trailers      http.Header
 
 	chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data.
 	buffer            []byte
 	offset            int
 	err               error
+	debug             bool // Print details on failure. Add your own if more are needed.
 }
 
 func (cr *s3ChunkedReader) Close() (err error) {
@@ -214,6 +262,19 @@ const maxChunkSize = 16 << 20 // 16 MiB
 // Read - implements `io.Reader`, which transparently decodes
 // the incoming AWS Signature V4 streaming signature.
 func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
+	if cr.err != nil {
+		if cr.debug {
+			fmt.Printf("s3ChunkedReader: Returning err: %v (%T)\n", cr.err, cr.err)
+		}
+		return 0, cr.err
+	}
+	defer func() {
+		if err != nil && err != io.EOF {
+			if cr.debug {
+				fmt.Println("Read err:", err)
+			}
+		}
+	}()
 	// First, if there is any unread data, copy it to the client
 	// provided buffer.
 	if cr.offset > 0 {
@@ -319,8 +380,43 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
 		cr.err = err
 		return n, cr.err
 	}
+
+	// Once we have read the entire chunk successfully, we verify
+	// that the received signature matches our computed signature.
+	cr.chunkSHA256Writer.Write(cr.buffer)
+	newSignature := cr.getChunkSignature()
+	if !compareSignatureV4(string(signature[16:]), newSignature) {
+		cr.err = errSignatureMismatch
+		return n, cr.err
+	}
+	cr.seedSignature = newSignature
+	cr.chunkSHA256Writer.Reset()
+
+	// If the chunk size is zero we return io.EOF. As specified by AWS,
+	// only the last chunk is zero-sized.
+	if len(cr.buffer) == 0 {
+		if cr.debug {
+			fmt.Println("EOF. Reading Trailers:", cr.trailers)
+		}
+		if cr.trailers != nil {
+			err = cr.readTrailers()
+			if cr.debug {
+				fmt.Println("trailers returned:", err, "now:", cr.trailers)
+			}
+			if err != nil {
+				cr.err = err
+				return 0, err
+			}
+		}
+		cr.err = io.EOF
+		return n, cr.err
+	}
+
 	b, err = cr.reader.ReadByte()
 	if b != '\r' || err != nil {
+		if cr.debug {
+			fmt.Printf("want %q, got %q\n", "\r", string(b))
+		}
 		cr.err = errMalformedEncoding
 		return n, cr.err
 	}
@@ -333,33 +429,133 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
 		return n, cr.err
 	}
 	if b != '\n' {
+		if cr.debug {
+			fmt.Printf("want %q, got %q\n", "\r", string(b))
+		}
 		cr.err = errMalformedEncoding
 		return n, cr.err
 	}
 
-	// Once we have read the entire chunk successfully, we verify
-	// that the received signature matches our computed signature.
-	cr.chunkSHA256Writer.Write(cr.buffer)
-	newSignature := getChunkSignature(cr.cred, cr.seedSignature, cr.region, cr.seedDate, hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil)))
-	if !compareSignatureV4(string(signature[16:]), newSignature) {
-		cr.err = errSignatureMismatch
-		return n, cr.err
-	}
-	cr.seedSignature = newSignature
-	cr.chunkSHA256Writer.Reset()
-
-	// If the chunk size is zero we return io.EOF. As specified by AWS,
-	// only the last chunk is zero-sized.
-	if size == 0 {
-		cr.err = io.EOF
-		return n, cr.err
-	}
-
 	cr.offset = copy(buf, cr.buffer)
 	n += cr.offset
 	return n, err
 }
 
+// readTrailers will read all trailers and populate cr.trailers with actual values.
+func (cr *s3ChunkedReader) readTrailers() error {
+	var valueBuffer bytes.Buffer
+	// Read value
+	for {
+		v, err := cr.reader.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+		}
+		if v != '\r' {
+			valueBuffer.WriteByte(v)
+			continue
+		}
+		// End of buffer, do not add to value.
+		v, err = cr.reader.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+		}
+		if v != '\n' {
+			return errMalformedEncoding
+		}
+		break
+	}
+
+	// Read signature
+	var signatureBuffer bytes.Buffer
+	for {
+		v, err := cr.reader.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+		}
+		if v != '\r' {
+			signatureBuffer.WriteByte(v)
+			continue
+		}
+		var tmp [3]byte
+		_, err = io.ReadFull(cr.reader, tmp[:])
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+		}
+		if string(tmp[:]) != "\n\r\n" {
+			if cr.debug {
+				fmt.Printf("signature, want %q, got %q", "\n\r\n", string(tmp[:]))
+			}
+			return errMalformedEncoding
+		}
+		// No need to write final newlines to buffer.
+		break
+	}
+
+	// Verify signature.
+	sig := signatureBuffer.Bytes()
+	if !bytes.HasPrefix(sig, []byte("x-amz-trailer-signature:")) {
+		if cr.debug {
+			fmt.Printf("prefix, want prefix %q, got %q", "x-amz-trailer-signature:", string(sig))
+		}
+		return errMalformedEncoding
+	}
+	sig = sig[len("x-amz-trailer-signature:"):]
+	sig = bytes.TrimSpace(sig)
+	cr.chunkSHA256Writer.Write(valueBuffer.Bytes())
+	wantSig := cr.getTrailerChunkSignature()
+	if !compareSignatureV4(string(sig), wantSig) {
+		if cr.debug {
+			fmt.Printf("signature, want: %q, got %q\nSignature buffer: %q\n", wantSig, string(sig), string(valueBuffer.Bytes()))
+		}
+		return errSignatureMismatch
+	}
+
+	// Parse trailers.
+	wantTrailers := make(map[string]struct{}, len(cr.trailers))
+	for k := range cr.trailers {
+		wantTrailers[strings.ToLower(k)] = struct{}{}
+	}
+	input := bufio.NewScanner(bytes.NewReader(valueBuffer.Bytes()))
+	for input.Scan() {
+		line := strings.TrimSpace(input.Text())
+		if line == "" {
+			continue
+		}
+		// Find first separator.
+		idx := strings.IndexByte(line, trailerKVSeparator[0])
+		if idx <= 0 || idx >= len(line) {
+			if cr.debug {
+				fmt.Printf("index, ':' not found in %q\n", line)
+			}
+			return errMalformedEncoding
+		}
+		key := line[:idx]
+		value := line[idx+1:]
+		if _, ok := wantTrailers[key]; !ok {
+			if cr.debug {
+				fmt.Printf("%q not found in %q\n", key, cr.trailers)
+			}
+			return errMalformedEncoding
+		}
+		cr.trailers.Set(key, value)
+		delete(wantTrailers, key)
+	}
+
+	// Check if we got all we want.
+	if len(wantTrailers) > 0 {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
 // readCRLF - check if reader only has '\r\n' CRLF character.
 // returns malformed encoding if it doesn't.
 func readCRLF(reader io.Reader) error {