MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

etag: add FromContentMD5 to parse content-md5 as ETag (#11688) 

This commit adds the `FromContentMD5` function to
parse a client-provided content-md5 as ETag.

Further, it also adds multipart ETag computation
for future needs.
This commit is contained in:
Andreas Auernhammer 2021-03-03 21:58:28 +01:00 committed by GitHub
parent 2c198ae7b6
commit f14cc6c943
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 137 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/auth-handler.go
View File

@ -36,6 +36,7 @@ import (
"github.com/minio/minio/pkg/auth"
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
"github.com/minio/minio/pkg/bucket/policy"
"github.com/minio/minio/pkg/etag"
"github.com/minio/minio/pkg/hash"
iampolicy "github.com/minio/minio/pkg/iam/policy"
)
@ -430,19 +431,14 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
return errCode
}
var (
err error
contentMD5, contentSHA256 []byte
)
// Extract 'Content-Md5' if present.
contentMD5, err = checkValidMD5(r.Header)
clientETag, err := etag.FromContentMD5(r.Header)
if err != nil {
return ErrInvalidDigest
}
// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
var contentSHA256 []byte
if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
if sha256Sum, ok := r.URL.Query()[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
contentSHA256, err = hex.DecodeString(sha256Sum[0])
@ -459,7 +455,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
// The verification happens implicit during reading.
reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1)
reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
if err != nil {
return toAPIErrorCode(ctx, err)
}

11
cmd/object-handlers.go
View File

@ -19,7 +19,6 @@ package cmd
import (
"bufio"
"context"
"encoding/hex"
"encoding/xml"
"fmt"
"io"
@ -1414,8 +1413,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
}
}
// Get Content-Md5 sent by client and verify if valid
md5Bytes, err := checkValidMD5(r.Header)
clientETag, err := etag.FromContentMD5(r.Header)
if err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
return
@ -1469,7 +1467,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
}
var (
md5hex = hex.EncodeToString(md5Bytes)
md5hex = clientETag.String()
sha256hex = ""
reader io.Reader = r.Body
s3Err APIErrorCode
@ -2165,8 +2163,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
return
}
// get Content-Md5 sent by client and verify if valid
md5Bytes, err := checkValidMD5(r.Header)
clientETag, err := etag.FromContentMD5(r.Header)
if err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
return
@ -2217,7 +2214,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
}
var (
md5hex = hex.EncodeToString(md5Bytes)
md5hex = clientETag.String()
sha256hex = ""
reader io.Reader = r.Body
s3Error APIErrorCode

13
cmd/utils.go
View File

@ -20,7 +20,6 @@ import (
"bytes"
"context"
"crypto/tls"
"encoding/base64"
"encoding/json"
"encoding/xml"
"errors"
@ -143,18 +142,6 @@ func xmlDecoder(body io.Reader, v interface{}, size int64) error {
return d.Decode(v)
}
// checkValidMD5 - verify if valid md5, returns md5 in bytes.
func checkValidMD5(h http.Header) ([]byte, error) {
md5B64, ok := h[xhttp.ContentMD5]
if ok {
if md5B64[0] == "" {
return nil, fmt.Errorf("Content-Md5 header set to empty value")
}
return base64.StdEncoding.Strict().DecodeString(md5B64[0])
}
return []byte{}, nil
}
// hasContentMD5 returns true if Content-MD5 header is set.
func hasContentMD5(h http.Header) bool {
_, ok := h[xhttp.ContentMD5]

46
pkg/etag/etag.go
View File

@ -105,6 +105,8 @@ package etag
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
@ -176,6 +178,50 @@ var _ Tagger = ETag{} // compiler check
// the Tagger interface.
func (e ETag) ETag() ETag { return e }
// FromContentMD5 decodes and returns the Content-MD5
// as ETag, if set. If no Content-MD5 header is set
// it returns an empty ETag and no error.
func FromContentMD5(h http.Header) (ETag, error) {
v, ok := h["Content-Md5"]
if !ok {
return nil, nil
}
if v[0] == "" {
return nil, errors.New("etag: content-md5 is set but contains no value")
}
b, err := base64.StdEncoding.Strict().DecodeString(v[0])
if err != nil {
return nil, err
}
if len(b) != md5.Size {
return nil, errors.New("etag: invalid content-md5")
}
return ETag(b), nil
}
// Multipart computes an S3 multipart ETag given a list of
// S3 singlepart ETags. It returns nil if the list of
// ETags is empty.
//
// Any encrypted or multipart ETag will be ignored and not
// used to compute the returned ETag.
func Multipart(etags ...ETag) ETag {
if len(etags) == 0 {
return nil
}
var n int64
h := md5.New()
for _, etag := range etags {
if !etag.IsMultipart() && !etag.IsEncrypted() {
h.Write(etag)
n++
}
}
etag := append(h.Sum(nil), '-')
return strconv.AppendInt(etag, n, 10)
}
// Set adds the ETag to the HTTP headers. It overwrites any
// existing ETag entry.
//

83
pkg/etag/etag_test.go
View File

@ -17,6 +17,7 @@ package etag
import (
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
)
@ -137,3 +138,85 @@ func TestReader(t *testing.T) {
}
}
}
var multipartTests = []struct { // Test cases have been generated using AWS S3
ETags []ETag
Multipart ETag
}{
{
ETags: []ETag{},
Multipart: ETag{},
},
{
ETags: []ETag{must("b10a8db164e0754105b7a99be72e3fe5")},
Multipart: must("7b976cc68452e003eec7cb0eb631a19a-1"),
},
{
ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6")},
Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"),
},
{
ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("a096eb5968d607c2975fb2c4af9ab225"), must("b10a8db164e0754105b7a99be72e3fe5")},
Multipart: must("9a0d1febd9265f59f368ceb652770bc2-3"),
},
{ // Check that multipart ETags are ignored
ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("ceb8853ddc5086cc4ab9e149f8f09c88-1")},
Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"),
},
{ // Check that encrypted ETags are ignored
ETags: []ETag{
must("90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941"),
must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6"),
},
Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"),
},
}
func TestMultipart(t *testing.T) {
for i, test := range multipartTests {
if multipart := Multipart(test.ETags...); !Equal(multipart, test.Multipart) {
t.Fatalf("Test %d: got %q - want %q", i, multipart, test.Multipart)
}
}
}
var fromContentMD5Tests = []struct {
Header http.Header
ETag ETag
ShouldFail bool
}{
{Header: http.Header{}, ETag: nil}, // 0
{Header: http.Header{"Content-Md5": []string{"1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: must("d41d8cd98f00b204e9800998ecf8427e")}, // 1
{Header: http.Header{"Content-Md5": []string{"sQqNsWTgdUEFt6mb5y4/5Q=="}}, ETag: must("b10a8db164e0754105b7a99be72e3fe5")}, // 2
{Header: http.Header{"Content-MD5": []string{"1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: nil}, // 3 (Content-MD5 vs Content-Md5)
{Header: http.Header{"Content-Md5": []string{"sQqNsWTgdUEFt6mb5y4/5Q==", "1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: must("b10a8db164e0754105b7a99be72e3fe5")}, // 4
{Header: http.Header{"Content-Md5": []string{""}}, ShouldFail: true}, // 5 (empty value)
{Header: http.Header{"Content-Md5": []string{"", "sQqNsWTgdUEFt6mb5y4/5Q=="}}, ShouldFail: true}, // 6 (empty value)
{Header: http.Header{"Content-Md5": []string{"d41d8cd98f00b204e9800998ecf8427e"}}, ShouldFail: true}, // 7 (content-md5 is invalid b64 / of invalid length)
}
func TestFromContentMD5(t *testing.T) {
for i, test := range fromContentMD5Tests {
ETag, err := FromContentMD5(test.Header)
if err != nil && !test.ShouldFail {
t.Fatalf("Test %d: failed to convert Content-MD5 to ETag: %v", i, err)
}
if err == nil && test.ShouldFail {
t.Fatalf("Test %d: should have failed but succeeded", i)
}
if err == nil {
if !Equal(ETag, test.ETag) {
t.Fatalf("Test %d: got %q - want %q", i, ETag, test.ETag)
}
}
}
}
func must(s string) ETag {
t, err := Parse(s)
if err != nil {
panic(err)
}
return t
}