MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-10 14:09:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add object compression support (#6292)

Browse Source 
Add support for streaming (golang/LZ77/snappy) compression.
This commit is contained in:
Praveen raj Mani
2018-09-28 09:06:17 +05:30
committed by Nitish Tiwari
parent 5c765bc63e
commit ce9d36d954
57 changed files with 1321 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/hash/reader.go
View File

@@ -33,8 +33,9 @@ var errNestedReader = errors.New("Nesting of Reader detected, not allowed")
// Reader writes what it reads from an io.Reader to an MD5 and SHA256 hash.Hash.
// Reader verifies that the content of the io.Reader matches the expected checksums.
type Reader struct {
src io.Reader
size int64
src io.Reader
size int64
actualSize int64
md5sum, sha256sum []byte // Byte values of md5sum, sha256sum of client sent values.
md5Hash, sha256Hash hash.Hash
@@ -42,7 +43,7 @@ type Reader struct {
// NewReader returns a new hash Reader which computes the MD5 sum and
// SHA256 sum (if set) of the provided io.Reader at EOF.
func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string) (*Reader, error) {
func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) {
if _, ok := src.(*Reader); ok {
return nil, errNestedReader
}
@@ -71,6 +72,7 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string) (*Reader, er
size: size,
md5Hash: md5.New(),
sha256Hash: sha256Hash,
actualSize: actualSize,
}, nil
}
@@ -98,6 +100,10 @@ func (r *Reader) Read(p []byte) (n int, err error) {
// data.
func (r *Reader) Size() int64 { return r.size }
// ActualSize returns the pre-modified size of the object.
// DecompressedSize - For compressed objects.
func (r *Reader) ActualSize() int64 { return r.actualSize }
// MD5 - returns byte md5 value
func (r *Reader) MD5() []byte {
return r.md5sum

40
pkg/hash/reader_test.go
View File

@@ -26,7 +26,7 @@ import (
// Tests functions like Size(), MD5*(), SHA256*()
func TestHashReaderHelperMethods(t *testing.T) {
r, err := NewReader(bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589")
r, err := NewReader(bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4)
if err != nil {
t.Fatal(err)
}
@@ -46,6 +46,9 @@ func TestHashReaderHelperMethods(t *testing.T) {
if r.Size() != 4 {
t.Errorf("Expected size 4, got %d", r.Size())
}
if r.ActualSize() != 4 {
t.Errorf("Expected size 4, got %d", r.ActualSize())
}
expectedMD5, err := hex.DecodeString("e2fc714c4727ee9395f324cd2e7f331f")
if err != nil {
t.Fatal(err)
@@ -67,19 +70,22 @@ func TestHashReaderVerification(t *testing.T) {
testCases := []struct {
src io.Reader
size int64
actualSize int64
md5hex, sha256hex string
err error
}{
// Success, no checksum verification provided.
{
src: bytes.NewReader([]byte("abcd")),
size: 4,
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
},
// Failure md5 mismatch.
{
src: bytes.NewReader([]byte("abcd")),
size: 4,
md5hex: "d41d8cd98f00b204e9800998ecf8427f",
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
md5hex: "d41d8cd98f00b204e9800998ecf8427f",
err: BadDigest{
"d41d8cd98f00b204e9800998ecf8427f",
"e2fc714c4727ee9395f324cd2e7f331f",
@@ -87,9 +93,10 @@ func TestHashReaderVerification(t *testing.T) {
},
// Failure sha256 mismatch.
{
src: bytes.NewReader([]byte("abcd")),
size: 4,
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
err: SHA256Mismatch{
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589",
@@ -97,7 +104,7 @@ func TestHashReaderVerification(t *testing.T) {
},
}
for i, testCase := range testCases {
r, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex)
r, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
if err != nil {
t.Fatalf("Test %d: Initializing reader failed %s", i+1, err)
}
@@ -115,6 +122,7 @@ func TestHashReaderInvalidArguments(t *testing.T) {
testCases := []struct {
src io.Reader
size int64
actualSize int64
md5hex, sha256hex string
success bool
expectedErr error
@@ -123,6 +131,7 @@ func TestHashReaderInvalidArguments(t *testing.T) {
{
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
md5hex: "invalid-md5",
success: false,
expectedErr: BadDigest{},
@@ -131,6 +140,7 @@ func TestHashReaderInvalidArguments(t *testing.T) {
{
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
sha256hex: "invalid-sha256",
success: false,
expectedErr: SHA256Mismatch{},
@@ -139,19 +149,21 @@ func TestHashReaderInvalidArguments(t *testing.T) {
{
src: &Reader{src: bytes.NewReader([]byte("abcd"))},
size: 4,
actualSize: 4,
success: false,
expectedErr: errNestedReader,
},
// Expected inputs, NewReader() will succeed.
{
src: bytes.NewReader([]byte("abcd")),
size: 4,
success: true,
src: bytes.NewReader([]byte("abcd")),
size: 4,
actualSize: 4,
success: true,
},
}
for i, testCase := range testCases {
_, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex)
_, err := NewReader(testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize)
if err != nil && testCase.success {
t.Errorf("Test %d: Expected success, but got error %s instead", i+1, err)
}