s3v4: read and verify S3 signature v4 chunks separately (#11801)

This commit fixes a security issue in the signature v4 chunked
reader. Before, the reader returned unverified data to the caller
and would only verify the chunk signature once it has encountered
the end of the chunk payload.

Now, the chunk reader reads the entire chunk into an in-memory buffer,
verifies the signature and then returns data to the caller.

In general, this is a common security problem. We verifying data
streams, the verifier MUST NOT return data to the upper layers / its
callers as long as it has not verified the current data chunk / data
segment:
```
func (r *Reader) Read(buffer []byte) {
   if err := r.readNext(r.internalBuffer); err != nil {
      return err
   }
   if err := r.verify(r.internalBuffer); err != nil {
      return err
   }
   copy(buffer, r.internalBuffer)
}
```

cmd/streaming-signature-v4.go

@@ -166,7 +166,7 @@ func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, APIErrorCode) {
 		seedDate:          seedDate,
 		region:            region,
 		chunkSHA256Writer: sha256.New(),
-		state:             readChunkHeader,
+		buffer:            make([]byte, 64*1024),
 	}, ErrNone
 }
 
@@ -178,62 +178,13 @@ type s3ChunkedReader struct {
 	seedSignature string
 	seedDate      time.Time
 	region        string
-	state             chunkState
+
-	lastChunk         bool
-	chunkSignature    string
 	chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data.
-	n                 uint64    // Unread bytes in chunk
+	buffer            []byte
+	offset            int
 	err               error
 }
 
-// Read chunk reads the chunk token signature portion.
-func (cr *s3ChunkedReader) readS3ChunkHeader() {
-	// Read the first chunk line until CRLF.
-	var hexChunkSize, hexChunkSignature []byte
-	hexChunkSize, hexChunkSignature, cr.err = readChunkLine(cr.reader)
-	if cr.err != nil {
-		return
-	}
-	// <hex>;token=value - converts the hex into its uint64 form.
-	cr.n, cr.err = parseHexUint(hexChunkSize)
-	if cr.err != nil {
-		return
-	}
-	if cr.n == 0 {
-		cr.err = io.EOF
-	}
-	// Save the incoming chunk signature.
-	cr.chunkSignature = string(hexChunkSignature)
-}
-
-type chunkState int
-
-const (
-	readChunkHeader chunkState = iota
-	readChunkTrailer
-	readChunk
-	verifyChunk
-	eofChunk
-)
-
-func (cs chunkState) String() string {
-	stateString := ""
-	switch cs {
-	case readChunkHeader:
-		stateString = "readChunkHeader"
-	case readChunkTrailer:
-		stateString = "readChunkTrailer"
-	case readChunk:
-		stateString = "readChunk"
-	case verifyChunk:
-		stateString = "verifyChunk"
-	case eofChunk:
-		stateString = "eofChunk"
-
-	}
-	return stateString
-}
-
 func (cr *s3ChunkedReader) Close() (err error) {
 	return nil
 }
@@ -241,83 +192,165 @@ func (cr *s3ChunkedReader) Close() (err error) {
 // Read - implements `io.Reader`, which transparently decodes
 // the incoming AWS Signature V4 streaming signature.
 func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
-	for {
+	// First, if there is any unread data, copy it to the client
-		switch cr.state {
+	// provided buffer.
-		case readChunkHeader:
+	if cr.offset > 0 {
-			cr.readS3ChunkHeader()
+		n = copy(buf, cr.buffer[cr.offset:])
-			// If we're at the end of a chunk.
+		if n == len(buf) {
-			if cr.n == 0 && cr.err == io.EOF {
+			cr.offset += n
-				cr.state = readChunkTrailer
-				cr.lastChunk = true
-				continue
-			}
-			if cr.err != nil {
-				return 0, cr.err
-			}
-			cr.state = readChunk
-		case readChunkTrailer:
-			cr.err = readCRLF(cr.reader)
-			if cr.err != nil {
-				return 0, errMalformedEncoding
-			}
-			cr.state = verifyChunk
-		case readChunk:
-			// There is no more space left in the request buffer.
-			if len(buf) == 0 {
 			return n, nil
 		}
-			rbuf := buf
+		cr.offset = 0
-			// The request buffer is larger than the current chunk size.
+		buf = buf[n:]
-			// Read only the current chunk from the underlying reader.
-			if uint64(len(rbuf)) > cr.n {
-				rbuf = rbuf[:cr.n]
-			}
-			var n0 int
-			n0, cr.err = cr.reader.Read(rbuf)
-			if cr.err != nil {
-				// We have lesser than chunk size advertised in chunkHeader, this is 'unexpected'.
-				if cr.err == io.EOF {
-					cr.err = io.ErrUnexpectedEOF
-				}
-				return 0, cr.err
 	}
 
-			// Calculate sha256.
+	// Now, we read one chunk from the underlying reader.
-			cr.chunkSHA256Writer.Write(rbuf[:n0])
+	// A chunk has the following format:
-			// Update the bytes read into request buffer so far.
+	//   <chunk-size-as-hex> + ";chunk-signature=" + <signature-as-hex> + "\r\n" + <payload> + "\r\n"
-			n += n0
+	//
-			buf = buf[n0:]
+	// Frist, we read the chunk size but fail if it is larger
-			// Update bytes to be read of the current chunk before verifying chunk's signature.
+	// than 1 MB. We must not accept arbitrary large chunks.
-			cr.n -= uint64(n0)
+	// One 1 MB is a reasonable max limit.
-
+	//
-			// If we're at the end of a chunk.
+	// Then we read the signature and payload data. We compute the SHA256 checksum
-			if cr.n == 0 {
+	// of the payload and verify that it matches the expected signature value.
-				cr.state = readChunkTrailer
+	//
-				continue
+	// The last chunk is *always* 0-sized. So, we must only return io.EOF if we have encountered
+	// a chunk with a chunk size = 0. However, this chunk still has a signature and we must
+	// verify it.
+	const MaxSize = 1 << 20 // 1 MB
+	var size int
+	for {
+		b, err := cr.reader.ReadByte()
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
 		}
-		case verifyChunk:
+		if err != nil {
-			// Calculate the hashed chunk.
+			cr.err = err
-			hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))
+			return n, cr.err
-			// Calculate the chunk signature.
+		}
-			newSignature := getChunkSignature(cr.cred, cr.seedSignature, cr.region, cr.seedDate, hashedChunk)
+		if b == ';' { // separating character
-			if !compareSignatureV4(cr.chunkSignature, newSignature) {
+			break
-				// Chunk signature doesn't match we return signature does not match.
+		}
+
+		// Manually deserialize the size since AWS specified
+		// the chunk size to be of variable width. In particular,
+		// a size of 16 is encoded as `10` while a size of 64 KB
+		// is `10000`.
+		switch {
+		case b >= '0' && b <= '9':
+			size = size<<4 | int(b-'0')
+		case b >= 'a' && b <= 'f':
+			size = size<<4 | int(b-('a'-10))
+		case b >= 'A' && b <= 'F':
+			size = size<<4 | int(b-('A'-10))
+		default:
+			cr.err = errMalformedEncoding
+			return n, cr.err
+		}
+		if size > MaxSize {
+			cr.err = errMalformedEncoding
+			return n, cr.err
+		}
+	}
+
+	// Now, we read the signature of the following payload and expect:
+	//   chunk-signature=" + <signature-as-hex> + "\r\n"
+	//
+	// The signature is 64 bytes long (hex-encoded SHA256 hash) and
+	// starts with a 16 byte header: len("chunk-signature=") + 64 == 80.
+	var signature [80]byte
+	_, err = io.ReadFull(cr.reader, signature[:])
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil {
+		cr.err = err
+		return n, cr.err
+	}
+	if !bytes.HasPrefix(signature[:], []byte("chunk-signature=")) {
+		cr.err = errMalformedEncoding
+		return n, cr.err
+	}
+	b, err := cr.reader.ReadByte()
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil {
+		cr.err = err
+		return n, cr.err
+	}
+	if b != '\r' {
+		cr.err = errMalformedEncoding
+		return n, cr.err
+	}
+	b, err = cr.reader.ReadByte()
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil {
+		cr.err = err
+		return n, cr.err
+	}
+	if b != '\n' {
+		cr.err = errMalformedEncoding
+		return n, cr.err
+	}
+
+	if cap(cr.buffer) < size {
+		cr.buffer = make([]byte, size)
+	} else {
+		cr.buffer = cr.buffer[:size]
+	}
+
+	// Now, we read the payload and compute its SHA-256 hash.
+	_, err = io.ReadFull(cr.reader, cr.buffer)
+	if err == io.EOF && size != 0 {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil && err != io.EOF {
+		cr.err = err
+		return n, cr.err
+	}
+	b, err = cr.reader.ReadByte()
+	if b != '\r' {
+		cr.err = errMalformedEncoding
+		return n, cr.err
+	}
+	b, err = cr.reader.ReadByte()
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil {
+		cr.err = err
+		return n, cr.err
+	}
+	if b != '\n' {
+		cr.err = errMalformedEncoding
+		return n, cr.err
+	}
+
+	// Once we have read the entire chunk successfully, we verify
+	// that the received signature matches our computed signature.
+	cr.chunkSHA256Writer.Write(cr.buffer)
+	newSignature := getChunkSignature(cr.cred, cr.seedSignature, cr.region, cr.seedDate, hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil)))
+	if !compareSignatureV4(string(signature[16:]), newSignature) {
 		cr.err = errSignatureMismatch
-				return 0, cr.err
+		return n, cr.err
 	}
-			// Newly calculated signature becomes the seed for the next chunk
-			// this follows the chaining.
 	cr.seedSignature = newSignature
 	cr.chunkSHA256Writer.Reset()
-			if cr.lastChunk {
+
-				cr.state = eofChunk
+	// If the chunk size is zero we return io.EOF. As specified by AWS,
-			} else {
+	// only the last chunk is zero-sized.
-				cr.state = readChunkHeader
+	if size == 0 {
-			}
+		cr.err = io.EOF
-		case eofChunk:
+		return n, cr.err
-			return n, io.EOF
-		}
 	}
+
+	cr.offset = copy(buf, cr.buffer)
+	n += cr.offset
+	return n, err
 }
 
 // readCRLF - check if reader only has '\r\n' CRLF character.