MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 05:34:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Switch to Snappy -> S2 compression (#8189)

Browse Source
This commit is contained in:
Klaus Post
2019-09-25 23:08:24 -07:00
committed by Harshavardhana
parent be313f1758
commit ff726969aa
12 changed files with 224 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
cmd/object-api-utils_test.go
View File

@@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
* MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,9 +21,11 @@ import (
"io"
"net/http"
"reflect"
"strconv"
"testing"
"github.com/golang/snappy"
"github.com/klauspost/compress/s2"
"github.com/minio/minio/cmd/crypto"
)
// Tests validate bucket name.
@@ -298,10 +300,11 @@ func TestIsCompressed(t *testing.T) {
testCases := []struct {
objInfo ObjectInfo
result bool
err bool
}{
{
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV1,
"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
},
@@ -309,7 +312,35 @@ func TestIsCompressed(t *testing.T) {
},
{
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-XYZ": "golang/snappy/LZ77",
UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
},
result: true,
},
{
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": "unknown/compression/type",
"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
},
result: true,
err: true,
},
{
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2",
crypto.SSEIV: "yes",
},
},
result: true,
err: true,
},
{
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-XYZ": "klauspost/compress/s2",
"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
},
@@ -324,11 +355,21 @@ func TestIsCompressed(t *testing.T) {
},
}
for i, test := range testCases {
got := test.objInfo.IsCompressed()
if got != test.result {
t.Errorf("Test %d - expected %v but received %v",
i+1, test.result, got)
}
t.Run(strconv.Itoa(i), func(t *testing.T) {
got := test.objInfo.IsCompressed()
if got != test.result {
t.Errorf("IsCompressed: Expected %v but received %v",
test.result, got)
}
got, gErr := test.objInfo.IsCompressedOK()
if got != test.result {
t.Errorf("IsCompressedOK: Expected %v but received %v",
test.result, got)
}
if gErr != nil != test.err {
t.Errorf("IsCompressedOK: want error: %t, got error: %v", test.err, gErr)
}
})
}
}
@@ -367,6 +408,13 @@ func TestExcludeForCompression(t *testing.T) {
},
result: false,
},
{
object: "object",
header: http.Header{
"Content-Type": []string{"text/something"},
},
result: false,
},
}
for i, test := range testCases {
globalIsCompressionEnabled = true
@@ -422,7 +470,7 @@ func TestGetActualSize(t *testing.T) {
}{
{
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2",
"X-Minio-Internal-actual-size": "100000001",
"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
@@ -441,7 +489,7 @@ func TestGetActualSize(t *testing.T) {
},
{
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2",
"X-Minio-Internal-actual-size": "841",
"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
@@ -451,7 +499,7 @@ func TestGetActualSize(t *testing.T) {
},
{
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2",
"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
Parts: []ObjectPartInfo{},
@@ -540,7 +588,7 @@ func TestGetCompressedOffsets(t *testing.T) {
}
}
func TestSnappyCompressReader(t *testing.T) {
func TestS2CompressReader(t *testing.T) {
tests := []struct {
name string
data []byte
@@ -554,7 +602,8 @@ func TestSnappyCompressReader(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case
r := newSnappyCompressReader(bytes.NewReader(tt.data))
r := newS2CompressReader(bytes.NewReader(tt.data))
defer r.Close()
var rdrBuf bytes.Buffer
_, err := io.CopyBuffer(&rdrBuf, r, buf)
@@ -563,7 +612,7 @@ func TestSnappyCompressReader(t *testing.T) {
}
var stdBuf bytes.Buffer
w := snappy.NewBufferedWriter(&stdBuf)
w := s2.NewWriter(&stdBuf)
_, err = io.CopyBuffer(w, bytes.NewReader(tt.data), buf)
if err != nil {
t.Fatal(err)
@@ -582,7 +631,7 @@ func TestSnappyCompressReader(t *testing.T) {
}
var decBuf bytes.Buffer
decRdr := snappy.NewReader(&rdrBuf)
decRdr := s2.NewReader(&rdrBuf)
_, err = io.Copy(&decBuf, decRdr)
if err != nil {
t.Fatal(err)