mirror of
https://github.com/minio/minio.git
synced 2025-11-07 21:02:58 -05:00
rename all remaining packages to internal/ (#12418)
This is to ensure that there are no projects that try to import `minio/minio/pkg` into their own repo. Any such common packages should go to `https://github.com/minio/pkg`
This commit is contained in:
Harshavardhana
GitHub
333
internal/etag/etag.go
Normal file
333
internal/etag/etag.go
Normal file
@@ -0,0 +1,333 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Package etag provides an implementation of S3 ETags.
|
||||
//
|
||||
// Each S3 object has an associated ETag that can be
|
||||
// used to e.g. quickly compare objects or check whether
|
||||
// the content of an object has changed.
|
||||
//
|
||||
// In general, an S3 ETag is an MD5 checksum of the object
|
||||
// content. However, there are many exceptions to this rule.
|
||||
//
|
||||
//
|
||||
// Single-part Upload
|
||||
//
|
||||
// In case of a basic single-part PUT operation - without server
|
||||
// side encryption or object compression - the ETag of an object
|
||||
// is its content MD5.
|
||||
//
|
||||
//
|
||||
// Multi-part Upload
|
||||
//
|
||||
// The ETag of an object does not correspond to its content MD5
|
||||
// when the object is uploaded in multiple parts via the S3
|
||||
// multipart API. Instead, S3 first computes a MD5 of each part:
|
||||
// e1 := MD5(part-1)
|
||||
// e2 := MD5(part-2)
|
||||
// ...
|
||||
// eN := MD5(part-N)
|
||||
//
|
||||
// Then, the ETag of the object is computed as MD5 of all individual
|
||||
// part checksums. S3 also encodes the number of parts into the ETag
|
||||
// by appending a -<number-of-parts> at the end:
|
||||
// ETag := MD5(e1 || e2 || e3 ... || eN) || -N
|
||||
//
|
||||
// For example: ceb8853ddc5086cc4ab9e149f8f09c88-5
|
||||
//
|
||||
// However, this scheme is only used for multipart objects that are
|
||||
// not encrypted.
|
||||
//
|
||||
// Server-side Encryption
|
||||
//
|
||||
// S3 specifies three types of server-side-encryption - SSE-C, SSE-S3
|
||||
// and SSE-KMS - with different semantics w.r.t. ETags.
|
||||
// In case of SSE-S3, the ETag of an object is computed the same as
|
||||
// for single resp. multipart plaintext objects. In particular,
|
||||
// the ETag of a singlepart SSE-S3 object is its content MD5.
|
||||
//
|
||||
// In case of SSE-C and SSE-KMS, the ETag of an object is computed
|
||||
// differently. For singlepart uploads the ETag is not the content
|
||||
// MD5 of the object. For multipart uploads the ETag is also not
|
||||
// the MD5 of the individual part checksums but it still contains
|
||||
// the number of parts as suffix.
|
||||
//
|
||||
// Instead, the ETag is kind of unpredictable for S3 clients when
|
||||
// an object is encrypted using SSE-C or SSE-KMS. Maybe AWS S3
|
||||
// computes the ETag as MD5 of the encrypted content but there is
|
||||
// no way to verify this assumption since the encryption happens
|
||||
// inside AWS S3.
|
||||
// Therefore, S3 clients must not make any assumption about ETags
|
||||
// in case of SSE-C or SSE-KMS except that the ETag is well-formed.
|
||||
//
|
||||
// To put all of this into a simple rule:
|
||||
// SSE-S3 : ETag == MD5
|
||||
// SSE-C : ETag != MD5
|
||||
// SSE-KMS: ETag != MD5
|
||||
//
|
||||
//
|
||||
// Encrypted ETags
|
||||
//
|
||||
// An S3 implementation has to remember the content MD5 of objects
|
||||
// in case of SSE-S3. However, storing the ETag of an encrypted
|
||||
// object in plaintext may reveal some information about the object.
|
||||
// For example, two objects with the same ETag are identical with
|
||||
// a very high probability.
|
||||
//
|
||||
// Therefore, an S3 implementation may encrypt an ETag before storing
|
||||
// it. In this case, the stored ETag may not be a well-formed S3 ETag.
|
||||
// For example, it can be larger due to a checksum added by authenticated
|
||||
// encryption schemes. Such an ETag must be decrypted before sent to an
|
||||
// S3 client.
|
||||
//
|
||||
//
|
||||
// S3 Clients
|
||||
//
|
||||
// There are many different S3 client implementations. Most of them
|
||||
// access the ETag by looking for the HTTP response header key "Etag".
|
||||
// However, some of them assume that the header key has to be "ETag"
|
||||
// (case-sensitive) and will fail otherwise.
|
||||
// Further, some clients require that the ETag value is a double-quoted
|
||||
// string. Therefore, this package provides dedicated functions for
|
||||
// adding and extracing the ETag to/from HTTP headers.
|
||||
package etag
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ETag is a single S3 ETag.
|
||||
//
|
||||
// An S3 ETag sometimes corresponds to the MD5 of
|
||||
// the S3 object content. However, when an object
|
||||
// is encrypted, compressed or uploaded using
|
||||
// the S3 multipart API then its ETag is not
|
||||
// necessarily the MD5 of the object content.
|
||||
//
|
||||
// For a more detailed description of S3 ETags
|
||||
// take a look at the package documentation.
|
||||
type ETag []byte
|
||||
|
||||
// String returns the string representation of the ETag.
|
||||
//
|
||||
// The returned string is a hex representation of the
|
||||
// binary ETag with an optional '-<part-number>' suffix.
|
||||
func (e ETag) String() string {
|
||||
if e.IsMultipart() {
|
||||
return hex.EncodeToString(e[:16]) + string(e[16:])
|
||||
}
|
||||
return hex.EncodeToString(e)
|
||||
}
|
||||
|
||||
// IsEncrypted reports whether the ETag is encrypted.
|
||||
func (e ETag) IsEncrypted() bool {
|
||||
return len(e) > 16 && !bytes.ContainsRune(e, '-')
|
||||
}
|
||||
|
||||
// IsMultipart reports whether the ETag belongs to an
|
||||
// object that has been uploaded using the S3 multipart
|
||||
// API.
|
||||
// An S3 multipart ETag has a -<part-number> suffix.
|
||||
func (e ETag) IsMultipart() bool {
|
||||
return len(e) > 16 && bytes.ContainsRune(e, '-')
|
||||
}
|
||||
|
||||
// Parts returns the number of object parts that are
|
||||
// referenced by this ETag. It returns 1 if the object
|
||||
// has been uploaded using the S3 singlepart API.
|
||||
//
|
||||
// Parts may panic if the ETag is an invalid multipart
|
||||
// ETag.
|
||||
func (e ETag) Parts() int {
|
||||
if !e.IsMultipart() {
|
||||
return 1
|
||||
}
|
||||
|
||||
n := bytes.IndexRune(e, '-')
|
||||
parts, err := strconv.Atoi(string(e[n+1:]))
|
||||
if err != nil {
|
||||
panic(err) // malformed ETag
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
var _ Tagger = ETag{} // compiler check
|
||||
|
||||
// ETag returns the ETag itself.
|
||||
//
|
||||
// By providing this method ETag implements
|
||||
// the Tagger interface.
|
||||
func (e ETag) ETag() ETag { return e }
|
||||
|
||||
// FromContentMD5 decodes and returns the Content-MD5
|
||||
// as ETag, if set. If no Content-MD5 header is set
|
||||
// it returns an empty ETag and no error.
|
||||
func FromContentMD5(h http.Header) (ETag, error) {
|
||||
v, ok := h["Content-Md5"]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
if v[0] == "" {
|
||||
return nil, errors.New("etag: content-md5 is set but contains no value")
|
||||
}
|
||||
b, err := base64.StdEncoding.Strict().DecodeString(v[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(b) != md5.Size {
|
||||
return nil, errors.New("etag: invalid content-md5")
|
||||
}
|
||||
return ETag(b), nil
|
||||
}
|
||||
|
||||
// Multipart computes an S3 multipart ETag given a list of
|
||||
// S3 singlepart ETags. It returns nil if the list of
|
||||
// ETags is empty.
|
||||
//
|
||||
// Any encrypted or multipart ETag will be ignored and not
|
||||
// used to compute the returned ETag.
|
||||
func Multipart(etags ...ETag) ETag {
|
||||
if len(etags) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var n int64
|
||||
h := md5.New()
|
||||
for _, etag := range etags {
|
||||
if !etag.IsMultipart() && !etag.IsEncrypted() {
|
||||
h.Write(etag)
|
||||
n++
|
||||
}
|
||||
}
|
||||
etag := append(h.Sum(nil), '-')
|
||||
return strconv.AppendInt(etag, n, 10)
|
||||
}
|
||||
|
||||
// Set adds the ETag to the HTTP headers. It overwrites any
|
||||
// existing ETag entry.
|
||||
//
|
||||
// Due to legacy S3 clients, that make incorrect assumptions
|
||||
// about HTTP headers, Set should be used instead of
|
||||
// http.Header.Set(...). Otherwise, some S3 clients will not
|
||||
// able to extract the ETag.
|
||||
func Set(etag ETag, h http.Header) {
|
||||
// Some (broken) S3 clients expect the ETag header to
|
||||
// literally "ETag" - not "Etag". Further, some clients
|
||||
// expect an ETag in double quotes. Therefore, we set the
|
||||
// ETag directly as map entry instead of using http.Header.Set
|
||||
h["ETag"] = []string{`"` + etag.String() + `"`}
|
||||
}
|
||||
|
||||
// Get extracts and parses an ETag from the given HTTP headers.
|
||||
// It returns an error when the HTTP headers do not contain
|
||||
// an ETag entry or when the ETag is malformed.
|
||||
//
|
||||
// Get only accepts AWS S3 compatible ETags - i.e. no
|
||||
// encrypted ETags - and therefore is stricter than Parse.
|
||||
func Get(h http.Header) (ETag, error) {
|
||||
const strict = true
|
||||
if v := h.Get("Etag"); v != "" {
|
||||
return parse(v, strict)
|
||||
}
|
||||
v, ok := h["ETag"]
|
||||
if !ok || len(v) == 0 {
|
||||
return nil, errors.New("etag: HTTP header does not contain an ETag")
|
||||
}
|
||||
return parse(v[0], strict)
|
||||
}
|
||||
|
||||
// Equal returns true if and only if the two ETags are
|
||||
// identical.
|
||||
func Equal(a, b ETag) bool { return bytes.Equal(a, b) }
|
||||
|
||||
// Parse parses s as an S3 ETag, returning the result.
|
||||
// The string can be an encrypted, singlepart
|
||||
// or multipart S3 ETag. It returns an error if s is
|
||||
// not a valid textual representation of an ETag.
|
||||
func Parse(s string) (ETag, error) {
|
||||
const strict = false
|
||||
return parse(s, strict)
|
||||
}
|
||||
|
||||
// parse parse s as an S3 ETag, returning the result.
|
||||
// It operates in one of two modes:
|
||||
// - strict
|
||||
// - non-strict
|
||||
//
|
||||
// In strict mode, parse only accepts ETags that
|
||||
// are AWS S3 compatible. In particular, an AWS
|
||||
// S3 ETag always consists of a 128 bit checksum
|
||||
// value and an optional -<part-number> suffix.
|
||||
// Therefore, s must have the following form in
|
||||
// strict mode: <32-hex-characters>[-<integer>]
|
||||
//
|
||||
// In non-strict mode, parse also accepts ETags
|
||||
// that are not AWS S3 compatible - e.g. encrypted
|
||||
// ETags.
|
||||
func parse(s string, strict bool) (ETag, error) {
|
||||
// An S3 ETag may be a double-quoted string.
|
||||
// Therefore, we remove double quotes at the
|
||||
// start and end, if any.
|
||||
if strings.HasPrefix(s, `"`) && strings.HasSuffix(s, `"`) {
|
||||
s = s[1 : len(s)-1]
|
||||
}
|
||||
|
||||
// An S3 ETag may be a multipart ETag that
|
||||
// contains a '-' followed by a number.
|
||||
// If the ETag does not a '-' is is either
|
||||
// a singlepart or encrypted ETag.
|
||||
n := strings.IndexRune(s, '-')
|
||||
if n == -1 {
|
||||
etag, err := hex.DecodeString(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strict && len(etag) != 16 { // AWS S3 ETags are always 128 bit long
|
||||
return nil, fmt.Errorf("etag: invalid length %d", len(etag))
|
||||
}
|
||||
return ETag(etag), nil
|
||||
}
|
||||
|
||||
prefix, suffix := s[:n], s[n:]
|
||||
if len(prefix) != 32 {
|
||||
return nil, fmt.Errorf("etag: invalid prefix length %d", len(prefix))
|
||||
}
|
||||
if len(suffix) <= 1 {
|
||||
return nil, errors.New("etag: suffix is not a part number")
|
||||
}
|
||||
|
||||
etag, err := hex.DecodeString(prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partNumber, err := strconv.Atoi(suffix[1:]) // suffix[0] == '-' Therefore, we start parsing at suffix[1]
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strict && (partNumber == 0 || partNumber > 10000) {
|
||||
return nil, fmt.Errorf("etag: invalid part number %d", partNumber)
|
||||
}
|
||||
return ETag(append(etag, suffix...)), nil
|
||||
}
|
||||
227
internal/etag/etag_test.go
Normal file
227
internal/etag/etag_test.go
Normal file
@@ -0,0 +1,227 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package etag
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var _ Tagger = Wrap(nil, nil).(Tagger) // runtime check that wrapReader implements Tagger
|
||||
|
||||
var parseTests = []struct {
|
||||
String string
|
||||
ETag ETag
|
||||
ShouldFail bool
|
||||
}{
|
||||
{String: "3b83ef96387f1465", ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101}}, // 0
|
||||
{String: "3b83ef96387f14655fc854ddc3c6bd57", ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101, 95, 200, 84, 221, 195, 198, 189, 87}}, // 1
|
||||
{String: `"3b83ef96387f14655fc854ddc3c6bd57"`, ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101, 95, 200, 84, 221, 195, 198, 189, 87}}, // 2
|
||||
{String: "ceb8853ddc5086cc4ab9e149f8f09c88-1", ETag: ETag{206, 184, 133, 61, 220, 80, 134, 204, 74, 185, 225, 73, 248, 240, 156, 136, 45, 49}}, // 3
|
||||
{String: `"ceb8853ddc5086cc4ab9e149f8f09c88-2"`, ETag: ETag{206, 184, 133, 61, 220, 80, 134, 204, 74, 185, 225, 73, 248, 240, 156, 136, 45, 50}}, // 4
|
||||
{ // 5
|
||||
String: "90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941",
|
||||
ETag: ETag{144, 64, 44, 120, 210, 220, 205, 222, 225, 233, 232, 98, 34, 206, 44, 99, 97, 103, 95, 53, 41, 210, 96, 0, 174, 46, 144, 15, 242, 22, 179, 203, 89, 225, 48, 224, 146, 216, 162, 152, 30, 119, 111, 77, 11, 214, 9, 65},
|
||||
},
|
||||
|
||||
{String: `"3b83ef96387f14655fc854ddc3c6bd57`, ShouldFail: true}, // 6
|
||||
{String: "ceb8853ddc5086cc4ab9e149f8f09c88-", ShouldFail: true}, // 7
|
||||
{String: "ceb8853ddc5086cc4ab9e149f8f09c88-2a", ShouldFail: true}, // 8
|
||||
{String: "ceb8853ddc5086cc4ab9e149f8f09c88-2-1", ShouldFail: true}, // 9
|
||||
{String: "90402c78d2dccddee1e9e86222ce2c-1", ShouldFail: true}, // 10
|
||||
{String: "90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941-1", ShouldFail: true}, // 11
|
||||
}
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
for i, test := range parseTests {
|
||||
etag, err := Parse(test.String)
|
||||
if err == nil && test.ShouldFail {
|
||||
t.Fatalf("Test %d: parse should have failed but succeeded", i)
|
||||
}
|
||||
if err != nil && !test.ShouldFail {
|
||||
t.Fatalf("Test %d: failed to parse ETag %q: %v", i, test.String, err)
|
||||
}
|
||||
if !Equal(etag, test.ETag) {
|
||||
t.Log([]byte(etag))
|
||||
t.Fatalf("Test %d: ETags don't match", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var stringTests = []struct {
|
||||
ETag ETag
|
||||
String string
|
||||
}{
|
||||
{ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101}, String: "3b83ef96387f1465"}, // 0
|
||||
{ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101, 95, 200, 84, 221, 195, 198, 189, 87}, String: "3b83ef96387f14655fc854ddc3c6bd57"}, // 1
|
||||
{ETag: ETag{206, 184, 133, 61, 220, 80, 134, 204, 74, 185, 225, 73, 248, 240, 156, 136, 45, 49}, String: "ceb8853ddc5086cc4ab9e149f8f09c88-1"}, // 2
|
||||
{ETag: ETag{206, 184, 133, 61, 220, 80, 134, 204, 74, 185, 225, 73, 248, 240, 156, 136, 45, 50}, String: "ceb8853ddc5086cc4ab9e149f8f09c88-2"}, // 3
|
||||
{ // 4
|
||||
ETag: ETag{144, 64, 44, 120, 210, 220, 205, 222, 225, 233, 232, 98, 34, 206, 44, 99, 97, 103, 95, 53, 41, 210, 96, 0, 174, 46, 144, 15, 242, 22, 179, 203, 89, 225, 48, 224, 146, 216, 162, 152, 30, 119, 111, 77, 11, 214, 9, 65},
|
||||
String: "90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941",
|
||||
},
|
||||
}
|
||||
|
||||
func TestString(t *testing.T) {
|
||||
for i, test := range stringTests {
|
||||
s := test.ETag.String()
|
||||
if s != test.String {
|
||||
t.Fatalf("Test %d: got %s - want %s", i, s, test.String)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var equalTests = []struct {
|
||||
A string
|
||||
B string
|
||||
Equal bool
|
||||
}{
|
||||
{A: "3b83ef96387f14655fc854ddc3c6bd57", B: "3b83ef96387f14655fc854ddc3c6bd57", Equal: true}, // 0
|
||||
{A: "3b83ef96387f14655fc854ddc3c6bd57", B: `"3b83ef96387f14655fc854ddc3c6bd57"`, Equal: true}, // 1
|
||||
|
||||
{A: "3b83ef96387f14655fc854ddc3c6bd57", B: "3b83ef96387f14655fc854ddc3c6bd57-2", Equal: false}, // 2
|
||||
{A: "3b83ef96387f14655fc854ddc3c6bd57", B: "ceb8853ddc5086cc4ab9e149f8f09c88", Equal: false}, // 3
|
||||
}
|
||||
|
||||
func TestEqual(t *testing.T) {
|
||||
for i, test := range equalTests {
|
||||
A, err := Parse(test.A)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: %v", i, err)
|
||||
}
|
||||
B, err := Parse(test.B)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: %v", i, err)
|
||||
}
|
||||
if equal := Equal(A, B); equal != test.Equal {
|
||||
t.Fatalf("Test %d: got %v - want %v", i, equal, test.Equal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var readerTests = []struct { // Reference values computed by: echo <content> | md5sum
|
||||
Content string
|
||||
ETag ETag
|
||||
}{
|
||||
{
|
||||
Content: "", ETag: ETag{212, 29, 140, 217, 143, 0, 178, 4, 233, 128, 9, 152, 236, 248, 66, 126},
|
||||
},
|
||||
{
|
||||
Content: " ", ETag: ETag{114, 21, 238, 156, 125, 157, 194, 41, 210, 146, 26, 64, 232, 153, 236, 95},
|
||||
},
|
||||
{
|
||||
Content: "Hello World", ETag: ETag{177, 10, 141, 177, 100, 224, 117, 65, 5, 183, 169, 155, 231, 46, 63, 229},
|
||||
},
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
for i, test := range readerTests {
|
||||
reader := NewReader(strings.NewReader(test.Content), test.ETag)
|
||||
if _, err := io.Copy(ioutil.Discard, reader); err != nil {
|
||||
t.Fatalf("Test %d: read failed: %v", i, err)
|
||||
}
|
||||
if ETag := reader.ETag(); !Equal(ETag, test.ETag) {
|
||||
t.Fatalf("Test %d: ETag mismatch: got %q - want %q", i, ETag, test.ETag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var multipartTests = []struct { // Test cases have been generated using AWS S3
|
||||
ETags []ETag
|
||||
Multipart ETag
|
||||
}{
|
||||
{
|
||||
ETags: []ETag{},
|
||||
Multipart: ETag{},
|
||||
},
|
||||
{
|
||||
ETags: []ETag{must("b10a8db164e0754105b7a99be72e3fe5")},
|
||||
Multipart: must("7b976cc68452e003eec7cb0eb631a19a-1"),
|
||||
},
|
||||
{
|
||||
ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6")},
|
||||
Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"),
|
||||
},
|
||||
{
|
||||
ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("a096eb5968d607c2975fb2c4af9ab225"), must("b10a8db164e0754105b7a99be72e3fe5")},
|
||||
Multipart: must("9a0d1febd9265f59f368ceb652770bc2-3"),
|
||||
},
|
||||
{ // Check that multipart ETags are ignored
|
||||
ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("ceb8853ddc5086cc4ab9e149f8f09c88-1")},
|
||||
Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"),
|
||||
},
|
||||
{ // Check that encrypted ETags are ignored
|
||||
ETags: []ETag{
|
||||
must("90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941"),
|
||||
must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6"),
|
||||
},
|
||||
Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"),
|
||||
},
|
||||
}
|
||||
|
||||
func TestMultipart(t *testing.T) {
|
||||
for i, test := range multipartTests {
|
||||
if multipart := Multipart(test.ETags...); !Equal(multipart, test.Multipart) {
|
||||
t.Fatalf("Test %d: got %q - want %q", i, multipart, test.Multipart)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var fromContentMD5Tests = []struct {
|
||||
Header http.Header
|
||||
ETag ETag
|
||||
ShouldFail bool
|
||||
}{
|
||||
{Header: http.Header{}, ETag: nil}, // 0
|
||||
{Header: http.Header{"Content-Md5": []string{"1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: must("d41d8cd98f00b204e9800998ecf8427e")}, // 1
|
||||
{Header: http.Header{"Content-Md5": []string{"sQqNsWTgdUEFt6mb5y4/5Q=="}}, ETag: must("b10a8db164e0754105b7a99be72e3fe5")}, // 2
|
||||
{Header: http.Header{"Content-MD5": []string{"1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: nil}, // 3 (Content-MD5 vs Content-Md5)
|
||||
{Header: http.Header{"Content-Md5": []string{"sQqNsWTgdUEFt6mb5y4/5Q==", "1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: must("b10a8db164e0754105b7a99be72e3fe5")}, // 4
|
||||
|
||||
{Header: http.Header{"Content-Md5": []string{""}}, ShouldFail: true}, // 5 (empty value)
|
||||
{Header: http.Header{"Content-Md5": []string{"", "sQqNsWTgdUEFt6mb5y4/5Q=="}}, ShouldFail: true}, // 6 (empty value)
|
||||
{Header: http.Header{"Content-Md5": []string{"d41d8cd98f00b204e9800998ecf8427e"}}, ShouldFail: true}, // 7 (content-md5 is invalid b64 / of invalid length)
|
||||
}
|
||||
|
||||
func TestFromContentMD5(t *testing.T) {
|
||||
for i, test := range fromContentMD5Tests {
|
||||
ETag, err := FromContentMD5(test.Header)
|
||||
if err != nil && !test.ShouldFail {
|
||||
t.Fatalf("Test %d: failed to convert Content-MD5 to ETag: %v", i, err)
|
||||
}
|
||||
if err == nil && test.ShouldFail {
|
||||
t.Fatalf("Test %d: should have failed but succeeded", i)
|
||||
}
|
||||
if err == nil {
|
||||
if !Equal(ETag, test.ETag) {
|
||||
t.Fatalf("Test %d: got %q - want %q", i, ETag, test.ETag)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func must(s string) ETag {
|
||||
t, err := Parse(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
156
internal/etag/reader.go
Normal file
156
internal/etag/reader.go
Normal file
@@ -0,0 +1,156 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package etag
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Tagger is the interface that wraps the basic ETag method.
|
||||
type Tagger interface {
|
||||
ETag() ETag
|
||||
}
|
||||
|
||||
type wrapReader struct {
|
||||
io.Reader
|
||||
Tagger
|
||||
}
|
||||
|
||||
var _ Tagger = wrapReader{} // compiler check
|
||||
|
||||
// ETag returns the ETag of the underlying Tagger.
|
||||
func (r wrapReader) ETag() ETag {
|
||||
if r.Tagger == nil {
|
||||
return nil
|
||||
}
|
||||
return r.Tagger.ETag()
|
||||
}
|
||||
|
||||
// Wrap returns an io.Reader that reads from the wrapped
|
||||
// io.Reader and implements the Tagger interaface.
|
||||
//
|
||||
// If content implements Tagger then the returned Reader
|
||||
// returns ETag of the content. Otherwise, it returns
|
||||
// nil as ETag.
|
||||
//
|
||||
// Wrap provides an adapter for io.Reader implemetations
|
||||
// that don't implement the Tagger interface.
|
||||
// It is mainly used to provide a high-level io.Reader
|
||||
// access to the ETag computed by a low-level io.Reader:
|
||||
//
|
||||
// content := etag.NewReader(r.Body, nil)
|
||||
//
|
||||
// compressedContent := Compress(content)
|
||||
// encryptedContent := Encrypt(compressedContent)
|
||||
//
|
||||
// // Now, we need an io.Reader that can access
|
||||
// // the ETag computed over the content.
|
||||
// reader := etag.Wrap(encryptedContent, content)
|
||||
//
|
||||
func Wrap(wrapped, content io.Reader) io.Reader {
|
||||
if t, ok := content.(Tagger); ok {
|
||||
return wrapReader{
|
||||
Reader: wrapped,
|
||||
Tagger: t,
|
||||
}
|
||||
}
|
||||
return wrapReader{
|
||||
Reader: wrapped,
|
||||
}
|
||||
}
|
||||
|
||||
// A Reader wraps an io.Reader and computes the
|
||||
// MD5 checksum of the read content as ETag.
|
||||
//
|
||||
// Optionally, a Reader can also verify that
|
||||
// the computed ETag matches an expected value.
|
||||
// Therefore, it compares both ETags once the
|
||||
// underlying io.Reader returns io.EOF.
|
||||
// If the computed ETag does not match the
|
||||
// expected ETag then Read returns a VerifyError.
|
||||
//
|
||||
// Reader implements the Tagger interface.
|
||||
type Reader struct {
|
||||
src io.Reader
|
||||
|
||||
md5 hash.Hash
|
||||
checksum ETag
|
||||
|
||||
readN int64
|
||||
}
|
||||
|
||||
// NewReader returns a new Reader that computes the
|
||||
// MD5 checksum of the content read from r as ETag.
|
||||
//
|
||||
// If the provided etag is not nil the returned
|
||||
// Reader compares the etag with the computed
|
||||
// MD5 sum once the r returns io.EOF.
|
||||
func NewReader(r io.Reader, etag ETag) *Reader {
|
||||
if er, ok := r.(*Reader); ok {
|
||||
if er.readN == 0 && Equal(etag, er.checksum) {
|
||||
return er
|
||||
}
|
||||
}
|
||||
return &Reader{
|
||||
src: r,
|
||||
md5: md5.New(),
|
||||
checksum: etag,
|
||||
}
|
||||
}
|
||||
|
||||
// Read reads up to len(p) bytes from the underlying
|
||||
// io.Reader as specified by the io.Reader interface.
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
n, err := r.src.Read(p)
|
||||
r.readN += int64(n)
|
||||
r.md5.Write(p[:n])
|
||||
|
||||
if err == io.EOF && len(r.checksum) != 0 {
|
||||
if etag := r.ETag(); !Equal(etag, r.checksum) {
|
||||
return n, VerifyError{
|
||||
Expected: r.checksum,
|
||||
Computed: etag,
|
||||
}
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ETag returns the ETag of all the content read
|
||||
// so far. Reading more content changes the MD5
|
||||
// checksum. Therefore, calling ETag multiple
|
||||
// times may return different results.
|
||||
func (r *Reader) ETag() ETag {
|
||||
sum := r.md5.Sum(nil)
|
||||
return ETag(sum)
|
||||
}
|
||||
|
||||
// VerifyError is an error signaling that a
|
||||
// computed ETag does not match an expected
|
||||
// ETag.
|
||||
type VerifyError struct {
|
||||
Expected ETag
|
||||
Computed ETag
|
||||
}
|
||||
|
||||
func (v VerifyError) Error() string {
|
||||
return fmt.Sprintf("etag: expected ETag %q does not match computed ETag %q", v.Expected, v.Computed)
|
||||
}
|
||||
Reference in New Issue
Block a user