// Copyright (c) 2015-2021 MinIO, Inc. // // This file is part of MinIO Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . // Package cmd This file implements helper functions to validate Streaming AWS // Signature Version '4' authorization header. package cmd import ( "bufio" "bytes" "encoding/hex" "errors" "hash" "io" "net/http" "time" humanize "github.com/dustin/go-humanize" "github.com/minio/minio/internal/auth" "github.com/minio/minio/internal/hash/sha256" xhttp "github.com/minio/minio/internal/http" ) // Streaming AWS Signature Version '4' constants. const ( emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" signV4ChunkedAlgorithm = "AWS4-HMAC-SHA256-PAYLOAD" streamingContentEncoding = "aws-chunked" ) // getChunkSignature - get chunk signature. func getChunkSignature(cred auth.Credentials, seedSignature string, region string, date time.Time, hashedChunk string) string { // Calculate string to sign. stringToSign := signV4ChunkedAlgorithm + "\n" + date.Format(iso8601Format) + "\n" + getScope(date, region) + "\n" + seedSignature + "\n" + emptySHA256 + "\n" + hashedChunk // Get hmac signing key. signingKey := getSigningKey(cred.SecretKey, date, region, serviceS3) // Calculate signature. newSignature := getSignature(signingKey, stringToSign) return newSignature } // calculateSeedSignature - Calculate seed signature in accordance with // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html // // returns signature, error otherwise if the signature mismatches or any other // error while parsing and validating. func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature string, region string, date time.Time, errCode APIErrorCode) { // Copy request. req := *r // Save authorization header. v4Auth := req.Header.Get(xhttp.Authorization) // Parse signature version '4' header. signV4Values, errCode := parseSignV4(v4Auth, globalSite.Region, serviceS3) if errCode != ErrNone { return cred, "", "", time.Time{}, errCode } // Payload streaming. payload := streamingContentSHA256 // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' if payload != req.Header.Get(xhttp.AmzContentSha256) { return cred, "", "", time.Time{}, ErrContentSHA256Mismatch } // Extract all the signed headers along with its values. extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r) if errCode != ErrNone { return cred, "", "", time.Time{}, errCode } cred, _, errCode = checkKeyValid(r, signV4Values.Credential.accessKey) if errCode != ErrNone { return cred, "", "", time.Time{}, errCode } // Verify if region is valid. region = signV4Values.Credential.scope.region // Extract date, if not present throw error. var dateStr string if dateStr = req.Header.Get("x-amz-date"); dateStr == "" { if dateStr = r.Header.Get("Date"); dateStr == "" { return cred, "", "", time.Time{}, ErrMissingDateHeader } } // Parse date header. var err error date, err = time.Parse(iso8601Format, dateStr) if err != nil { return cred, "", "", time.Time{}, ErrMalformedDate } // Query string. queryStr := req.Form.Encode() // Get canonical request. canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method) // Get string to sign from canonical request. stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope()) // Get hmac signing key. signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, serviceS3) // Calculate signature. newSignature := getSignature(signingKey, stringToSign) // Verify if signature match. if !compareSignatureV4(newSignature, signV4Values.Signature) { return cred, "", "", time.Time{}, ErrSignatureDoesNotMatch } // Return caculated signature. return cred, newSignature, region, date, ErrNone } const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB // lineTooLong is generated as chunk header is bigger than 4KiB. var errLineTooLong = errors.New("header line too long") // malformed encoding is generated when chunk header is wrongly formed. var errMalformedEncoding = errors.New("malformed chunked encoding") // chunk is considered too big if its bigger than > 16MiB. var errChunkTooBig = errors.New("chunk too big: choose chunk size <= 16MiB") // newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r // out of HTTP "chunked" format before returning it. // The s3ChunkedReader returns io.EOF when the final 0-length chunk is read. // // NewChunkedReader is not needed by normal applications. The http package // automatically decodes chunking when reading response bodies. func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, APIErrorCode) { cred, seedSignature, region, seedDate, errCode := calculateSeedSignature(req) if errCode != ErrNone { return nil, errCode } return &s3ChunkedReader{ reader: bufio.NewReader(req.Body), cred: cred, seedSignature: seedSignature, seedDate: seedDate, region: region, chunkSHA256Writer: sha256.New(), buffer: make([]byte, 64*1024), }, ErrNone } // Represents the overall state that is required for decoding a // AWS Signature V4 chunked reader. type s3ChunkedReader struct { reader *bufio.Reader cred auth.Credentials seedSignature string seedDate time.Time region string chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data. buffer []byte offset int err error } func (cr *s3ChunkedReader) Close() (err error) { return nil } // Now, we read one chunk from the underlying reader. // A chunk has the following format: // // + ";chunk-signature=" + + "\r\n" + + "\r\n" // // First, we read the chunk size but fail if it is larger // than 16 MiB. We must not accept arbitrary large chunks. // One 16 MiB is a reasonable max limit. // // Then we read the signature and payload data. We compute the SHA256 checksum // of the payload and verify that it matches the expected signature value. // // The last chunk is *always* 0-sized. So, we must only return io.EOF if we have encountered // a chunk with a chunk size = 0. However, this chunk still has a signature and we must // verify it. const maxChunkSize = 16 << 20 // 16 MiB // Read - implements `io.Reader`, which transparently decodes // the incoming AWS Signature V4 streaming signature. func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { // First, if there is any unread data, copy it to the client // provided buffer. if cr.offset > 0 { n = copy(buf, cr.buffer[cr.offset:]) if n == len(buf) { cr.offset += n return n, nil } cr.offset = 0 buf = buf[n:] } var size int for { b, err := cr.reader.ReadByte() if err == io.EOF { err = io.ErrUnexpectedEOF } if err != nil { cr.err = err return n, cr.err } if b == ';' { // separating character break } // Manually deserialize the size since AWS specified // the chunk size to be of variable width. In particular, // a size of 16 is encoded as `10` while a size of 64 KB // is `10000`. switch { case b >= '0' && b <= '9': size = size<<4 | int(b-'0') case b >= 'a' && b <= 'f': size = size<<4 | int(b-('a'-10)) case b >= 'A' && b <= 'F': size = size<<4 | int(b-('A'-10)) default: cr.err = errMalformedEncoding return n, cr.err } if size > maxChunkSize { cr.err = errChunkTooBig return n, cr.err } } // Now, we read the signature of the following payload and expect: // chunk-signature=" + + "\r\n" // // The signature is 64 bytes long (hex-encoded SHA256 hash) and // starts with a 16 byte header: len("chunk-signature=") + 64 == 80. var signature [80]byte _, err = io.ReadFull(cr.reader, signature[:]) if err == io.EOF { err = io.ErrUnexpectedEOF } if err != nil { cr.err = err return n, cr.err } if !bytes.HasPrefix(signature[:], []byte("chunk-signature=")) { cr.err = errMalformedEncoding return n, cr.err } b, err := cr.reader.ReadByte() if err == io.EOF { err = io.ErrUnexpectedEOF } if err != nil { cr.err = err return n, cr.err } if b != '\r' { cr.err = errMalformedEncoding return n, cr.err } b, err = cr.reader.ReadByte() if err == io.EOF { err = io.ErrUnexpectedEOF } if err != nil { cr.err = err return n, cr.err } if b != '\n' { cr.err = errMalformedEncoding return n, cr.err } if cap(cr.buffer) < size { cr.buffer = make([]byte, size) } else { cr.buffer = cr.buffer[:size] } // Now, we read the payload and compute its SHA-256 hash. _, err = io.ReadFull(cr.reader, cr.buffer) if err == io.EOF && size != 0 { err = io.ErrUnexpectedEOF } if err != nil && err != io.EOF { cr.err = err return n, cr.err } b, err = cr.reader.ReadByte() if b != '\r' { cr.err = errMalformedEncoding return n, cr.err } b, err = cr.reader.ReadByte() if err == io.EOF { err = io.ErrUnexpectedEOF } if err != nil { cr.err = err return n, cr.err } if b != '\n' { cr.err = errMalformedEncoding return n, cr.err } // Once we have read the entire chunk successfully, we verify // that the received signature matches our computed signature. cr.chunkSHA256Writer.Write(cr.buffer) newSignature := getChunkSignature(cr.cred, cr.seedSignature, cr.region, cr.seedDate, hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))) if !compareSignatureV4(string(signature[16:]), newSignature) { cr.err = errSignatureMismatch return n, cr.err } cr.seedSignature = newSignature cr.chunkSHA256Writer.Reset() // If the chunk size is zero we return io.EOF. As specified by AWS, // only the last chunk is zero-sized. if size == 0 { cr.err = io.EOF return n, cr.err } cr.offset = copy(buf, cr.buffer) n += cr.offset return n, err } // readCRLF - check if reader only has '\r\n' CRLF character. // returns malformed encoding if it doesn't. func readCRLF(reader io.Reader) error { buf := make([]byte, 2) _, err := io.ReadFull(reader, buf[:2]) if err != nil { return err } if buf[0] != '\r' || buf[1] != '\n' { return errMalformedEncoding } return nil } // Read a line of bytes (up to \n) from b. // Give up if the line exceeds maxLineLength. // The returned bytes are owned by the bufio.Reader // so they are only valid until the next bufio read. func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) { buf, err := b.ReadSlice('\n') if err != nil { // We always know when EOF is coming. // If the caller asked for a line, there should be a line. if err == io.EOF { err = io.ErrUnexpectedEOF } else if err == bufio.ErrBufferFull { err = errLineTooLong } return nil, nil, err } if len(buf) >= maxLineLength { return nil, nil, errLineTooLong } // Parse s3 specific chunk extension and fetch the values. hexChunkSize, hexChunkSignature := parseS3ChunkExtension(buf) return hexChunkSize, hexChunkSignature, nil } // trimTrailingWhitespace - trim trailing white space. func trimTrailingWhitespace(b []byte) []byte { for len(b) > 0 && isASCIISpace(b[len(b)-1]) { b = b[:len(b)-1] } return b } // isASCIISpace - is ascii space? func isASCIISpace(b byte) bool { return b == ' ' || b == '\t' || b == '\n' || b == '\r' } // Constant s3 chunk encoding signature. const s3ChunkSignatureStr = ";chunk-signature=" // parses3ChunkExtension removes any s3 specific chunk-extension from buf. // For example, // // "10000;chunk-signature=..." => "10000", "chunk-signature=..." func parseS3ChunkExtension(buf []byte) ([]byte, []byte) { buf = trimTrailingWhitespace(buf) semi := bytes.Index(buf, []byte(s3ChunkSignatureStr)) // Chunk signature not found, return the whole buffer. if semi == -1 { return buf, nil } return buf[:semi], parseChunkSignature(buf[semi:]) } // parseChunkSignature - parse chunk signature. func parseChunkSignature(chunk []byte) []byte { chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2) return chunkSplits[1] } // parse hex to uint64. func parseHexUint(v []byte) (n uint64, err error) { for i, b := range v { switch { case '0' <= b && b <= '9': b -= '0' case 'a' <= b && b <= 'f': b = b - 'a' + 10 case 'A' <= b && b <= 'F': b = b - 'A' + 10 default: return 0, errors.New("invalid byte in chunk length") } if i == 16 { return 0, errors.New("http chunk length too large") } n <<= 4 n |= uint64(b) } return }