MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

rename all remaining packages to internal/ (#12418)

Browse Source 
This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`
This commit is contained in:
Harshavardhana
2021-06-01 14:59:40 -07:00
committed by GitHub
parent bf87c4b1e4
commit 1f262daf6f
540 changed files with 757 additions and 778 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
internal/hash/errors.go Normal file
View File

@@ -0,0 +1,50 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package hash
import "fmt"
// SHA256Mismatch - when content sha256 does not match with what was sent from client.
type SHA256Mismatch struct {
ExpectedSHA256 string
CalculatedSHA256 string
}
func (e SHA256Mismatch) Error() string {
return "Bad sha256: Expected " + e.ExpectedSHA256 + " does not match calculated " + e.CalculatedSHA256
}
// BadDigest - Content-MD5 you specified did not match what we received.
type BadDigest struct {
ExpectedMD5 string
CalculatedMD5 string
}
func (e BadDigest) Error() string {
return "Bad digest: Expected " + e.ExpectedMD5 + " does not match calculated " + e.CalculatedMD5
}
// ErrSizeMismatch error size mismatch
type ErrSizeMismatch struct {
Want int64
Got int64
}
func (e ErrSizeMismatch) Error() string {
return fmt.Sprintf("Size mismatch: got %d, want %d", e.Got, e.Want)
}

228
internal/hash/reader.go Normal file
View File

@@ -0,0 +1,228 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package hash
import (
"bytes"
"encoding/base64"
"encoding/hex"
"errors"
"hash"
"io"
"github.com/minio/minio/internal/etag"
)
// A Reader wraps an io.Reader and computes the MD5 checksum
// of the read content as ETag. Optionally, it also computes
// the SHA256 checksum of the content.
//
// If the reference values for the ETag and content SHA26
// are not empty then it will check whether the computed
// match the reference values.
type Reader struct {
src io.Reader
bytesRead int64
size int64
actualSize int64
checksum etag.ETag
contentSHA256 []byte
sha256 hash.Hash
}
// NewReader returns a new Reader that wraps src and computes
// MD5 checksum of everything it reads as ETag.
//
// It also computes the SHA256 checksum of everything it reads
// if sha256Hex is not the empty string.
//
// If size resp. actualSize is unknown at the time of calling
// NewReader then it should be set to -1.
//
// NewReader may try merge the given size, MD5 and SHA256 values
// into src - if src is a Reader - to avoid computing the same
// checksums multiple times.
func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) {
MD5, err := hex.DecodeString(md5Hex)
if err != nil {
return nil, BadDigest{ // TODO(aead): Return an error that indicates that an invalid ETag has been specified
ExpectedMD5: md5Hex,
CalculatedMD5: "",
}
}
SHA256, err := hex.DecodeString(sha256Hex)
if err != nil {
return nil, SHA256Mismatch{ // TODO(aead): Return an error that indicates that an invalid Content-SHA256 has been specified
ExpectedSHA256: sha256Hex,
CalculatedSHA256: "",
}
}
// Merge the size, MD5 and SHA256 values if src is a Reader.
// The size may be set to -1 by callers if unknown.
if r, ok := src.(*Reader); ok {
if r.bytesRead > 0 {
return nil, errors.New("hash: already read from hash reader")
}
if len(r.checksum) != 0 && len(MD5) != 0 && !etag.Equal(r.checksum, etag.ETag(MD5)) {
return nil, BadDigest{
ExpectedMD5: r.checksum.String(),
CalculatedMD5: md5Hex,
}
}
if len(r.contentSHA256) != 0 && len(SHA256) != 0 && !bytes.Equal(r.contentSHA256, SHA256) {
return nil, SHA256Mismatch{
ExpectedSHA256: hex.EncodeToString(r.contentSHA256),
CalculatedSHA256: sha256Hex,
}
}
if r.size >= 0 && size >= 0 && r.size != size {
return nil, ErrSizeMismatch{Want: r.size, Got: size}
}
r.checksum = etag.ETag(MD5)
r.contentSHA256 = SHA256
if r.size < 0 && size >= 0 {
r.src = etag.Wrap(io.LimitReader(r.src, size), r.src)
r.size = size
}
if r.actualSize <= 0 && actualSize >= 0 {
r.actualSize = actualSize
}
return r, nil
}
if size >= 0 {
r := io.LimitReader(src, size)
if _, ok := src.(etag.Tagger); !ok {
src = etag.NewReader(r, etag.ETag(MD5))
} else {
src = etag.Wrap(r, src)
}
} else if _, ok := src.(etag.Tagger); !ok {
src = etag.NewReader(src, etag.ETag(MD5))
}
var hash hash.Hash
if len(SHA256) != 0 {
hash = newSHA256()
}
return &Reader{
src: src,
size: size,
actualSize: actualSize,
checksum: etag.ETag(MD5),
contentSHA256: SHA256,
sha256: hash,
}, nil
}
func (r *Reader) Read(p []byte) (int, error) {
n, err := r.src.Read(p)
r.bytesRead += int64(n)
if r.sha256 != nil {
r.sha256.Write(p[:n])
}
if err == io.EOF { // Verify content SHA256, if set.
if r.sha256 != nil {
if sum := r.sha256.Sum(nil); !bytes.Equal(r.contentSHA256, sum) {
return n, SHA256Mismatch{
ExpectedSHA256: hex.EncodeToString(r.contentSHA256),
CalculatedSHA256: hex.EncodeToString(sum),
}
}
}
}
if err != nil && err != io.EOF {
if v, ok := err.(etag.VerifyError); ok {
return n, BadDigest{
ExpectedMD5: v.Expected.String(),
CalculatedMD5: v.Computed.String(),
}
}
}
return n, err
}
// Size returns the absolute number of bytes the Reader
// will return during reading. It returns -1 for unlimited
// data.
func (r *Reader) Size() int64 { return r.size }
// ActualSize returns the pre-modified size of the object.
// DecompressedSize - For compressed objects.
func (r *Reader) ActualSize() int64 { return r.actualSize }
// ETag returns the ETag computed by an underlying etag.Tagger.
// If the underlying io.Reader does not implement etag.Tagger
// it returns nil.
func (r *Reader) ETag() etag.ETag {
if t, ok := r.src.(etag.Tagger); ok {
return t.ETag()
}
return nil
}
// MD5 returns the MD5 checksum set as reference value.
//
// It corresponds to the checksum that is expected and
// not the actual MD5 checksum of the content.
// Therefore, refer to MD5Current.
func (r *Reader) MD5() []byte {
return r.checksum
}
// MD5Current returns the MD5 checksum of the content
// that has been read so far.
//
// Calling MD5Current again after reading more data may
// result in a different checksum.
func (r *Reader) MD5Current() []byte {
return r.ETag()[:]
}
// SHA256 returns the SHA256 checksum set as reference value.
//
// It corresponds to the checksum that is expected and
// not the actual SHA256 checksum of the content.
func (r *Reader) SHA256() []byte {
return r.contentSHA256
}
// MD5HexString returns a hex representation of the MD5.
func (r *Reader) MD5HexString() string {
return hex.EncodeToString(r.checksum)
}
// MD5Base64String returns a hex representation of the MD5.
func (r *Reader) MD5Base64String() string {
return base64.StdEncoding.EncodeToString(r.checksum)
}
// SHA256HexString returns a hex representation of the SHA256.
func (r *Reader) SHA256HexString() string {
return hex.EncodeToString(r.contentSHA256)
}
var _ io.Closer = (*Reader)(nil) // compiler check
// Close and release resources.
func (r *Reader) Close() error { return nil }

306
internal/hash/reader_test.go Normal file
View File

@@ -0,0 +1,306 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package hash
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"testing"
)
// Tests functions like Size(), MD5*(), SHA256*()
func TestHashReaderHelperMethods(t *testing.T) {
r, err := NewReader(bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4)
if err != nil {
t.Fatal(err)
}
_, err = io.Copy(ioutil.Discard, r)
if err != nil {
t.Fatal(err)
}
if r.MD5HexString() != "e2fc714c4727ee9395f324cd2e7f331f" {
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", r.MD5HexString())
}
if r.SHA256HexString() != "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" {
t.Errorf("Expected sha256hex \"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589\", got %s", r.SHA256HexString())
}
if r.MD5Base64String() != "4vxxTEcn7pOV8yTNLn8zHw==" {
t.Errorf("Expected md5base64 \"4vxxTEcn7pOV8yTNLn8zHw==\", got \"%s\"", r.MD5Base64String())
}
if r.Size() != 4 {
t.Errorf("Expected size 4, got %d", r.Size())
}
if r.ActualSize() != 4 {
t.Errorf("Expected size 4, got %d", r.ActualSize())
}
expectedMD5, err := hex.DecodeString("e2fc714c4727ee9395f324cd2e7f331f")
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(r.MD5(), expectedMD5) {
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", r.MD5HexString())
}
if !bytes.Equal(r.MD5Current(), expectedMD5) {
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", hex.EncodeToString(r.MD5Current()))
}
expectedSHA256, err := hex.DecodeString("88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589")
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(r.SHA256(), expectedSHA256) {
t.Errorf("Expected md5hex \"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589\", got %s", r.SHA256HexString())
}
}
// Tests hash reader checksum verification.
func TestHashReaderVerification(t *testing.T) {
testCases := []struct {
desc string
src io.Reader
size int64
actualSize int64
md5hex, sha256hex string
err error
}{
{
desc: "Success, no checksum verification provided.",
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
},
{
desc: "Failure md5 mismatch.",
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
md5hex: "d41d8cd98f00b204e9800998ecf8427f",
err: BadDigest{
"d41d8cd98f00b204e9800998ecf8427f",
"e2fc714c4727ee9395f324cd2e7f331f",
},
},
{
desc: "Failure sha256 mismatch.",
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
err: SHA256Mismatch{
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
},
},
{
desc: "Nested hash reader NewReader() should merge.",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
size: 4,
actualSize: 4,
},
{
desc: "Incorrect sha256, nested",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
size: 4,
actualSize: 4,
sha256hex: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c",
err: SHA256Mismatch{
ExpectedSHA256: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c",
CalculatedSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
},
},
{
desc: "Correct sha256, nested",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
size: 4,
actualSize: 4,
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
},
{
desc: "Correct sha256, nested, truncated",
src: mustReader(t, bytes.NewReader([]byte("abcd-more-stuff-to-be ignored")), 4, "", "", 4),
size: 4,
actualSize: -1,
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
},
{
desc: "Correct sha256, nested, truncated, swapped",
src: mustReader(t, bytes.NewReader([]byte("abcd-more-stuff-to-be ignored")), 4, "", "", -1),
size: 4,
actualSize: -1,
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
},
{
desc: "Incorrect MD5, nested",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
size: 4,
actualSize: 4,
md5hex: "0773da587b322af3a8718cb418a715ce",
err: BadDigest{
ExpectedMD5: "0773da587b322af3a8718cb418a715ce",
CalculatedMD5: "e2fc714c4727ee9395f324cd2e7f331f",
},
},
{
desc: "Correct sha256, truncated",
src: bytes.NewReader([]byte("abcd-morethan-4-bytes")),
size: 4,
actualSize: 4,
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
},
{
desc: "Correct MD5, nested",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
size: 4,
actualSize: 4,
md5hex: "e2fc714c4727ee9395f324cd2e7f331f",
},
{
desc: "Correct MD5, truncated",
src: bytes.NewReader([]byte("abcd-morethan-4-bytes")),
size: 4,
actualSize: 4,
sha256hex: "",
md5hex: "e2fc714c4727ee9395f324cd2e7f331f",
},
{
desc: "Correct MD5, nested, truncated",
src: mustReader(t, bytes.NewReader([]byte("abcd-morestuff")), -1, "", "", -1),
size: 4,
actualSize: 4,
md5hex: "e2fc714c4727ee9395f324cd2e7f331f",
},
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
r, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
if err != nil {
t.Fatalf("Test %q: Initializing reader failed %s", testCase.desc, err)
}
_, err = io.Copy(ioutil.Discard, r)
if err != nil {
if err.Error() != testCase.err.Error() {
t.Errorf("Test %q: Expected error %s, got error %s", testCase.desc, testCase.err, err)
}
}
})
}
}
func mustReader(t *testing.T, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) *Reader {
r, err := NewReader(src, size, md5Hex, sha256Hex, actualSize)
if err != nil {
t.Fatal(err)
}
return r
}
// Tests NewReader() constructor with invalid arguments.
func TestHashReaderInvalidArguments(t *testing.T) {
testCases := []struct {
desc string
src io.Reader
size int64
actualSize int64
md5hex, sha256hex string
success bool
}{
{
desc: "Invalid md5sum NewReader() will fail.",
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
md5hex: "invalid-md5",
success: false,
},
{
desc: "Invalid sha256 NewReader() will fail.",
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
sha256hex: "invalid-sha256",
success: false,
},
{
desc: "Nested hash reader NewReader() should merge.",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4),
size: 4,
actualSize: 4,
success: true,
},
{
desc: "Mismatching sha256",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4),
size: 4,
actualSize: 4,
sha256hex: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c",
success: false,
},
{
desc: "Correct sha256",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4),
size: 4,
actualSize: 4,
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
success: true,
},
{
desc: "Mismatching MD5",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "", 4),
size: 4,
actualSize: 4,
md5hex: "0773da587b322af3a8718cb418a715ce",
success: false,
},
{
desc: "Correct MD5",
src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "", 4),
size: 4,
actualSize: 4,
md5hex: "e2fc714c4727ee9395f324cd2e7f331f",
success: true,
},
{
desc: "Nothing, all ok",
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
success: true,
},
{
desc: "Nested, size mismatch",
src: mustReader(t, bytes.NewReader([]byte("abcd-morestuff")), 4, "", "", -1),
size: 2,
actualSize: -1,
success: false,
},
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) {
_, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
if err != nil && testCase.success {
t.Errorf("Test %q: Expected success, but got error %s instead", testCase.desc, err)
}
if err == nil && !testCase.success {
t.Errorf("Test %q: Expected error, but got success", testCase.desc)
}
})
}
}

31
internal/hash/sha256_fips.go Normal file
View File

@@ -0,0 +1,31 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// +build fips
package hash
import (
"crypto/sha256"
"hash"
)
// newSHA256 returns a new hash.Hash computing the SHA256 checksum.
// The SHA256 implementation is FIPS 140-2 compliant when the
// boringcrypto branch of Go is used.
// Ref: https://github.com/golang/go/tree/dev.boringcrypto
func newSHA256() hash.Hash { return sha256.New() }

30
internal/hash/sha256_nofips.go Normal file
View File

@@ -0,0 +1,30 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// +build !fips
package hash
import (
"hash"
sha256 "github.com/minio/sha256-simd"
)
// newSHA256 returns a new hash.Hash computing the SHA256 checksum.
// The SHA256 implementation is not FIPS 140-2 compliant.
func newSHA256() hash.Hash { return sha256.New() }