mirror of
https://github.com/minio/minio.git
synced 2024-12-25 06:35:56 -05:00
5a80cbec2a
This PR adds pass-through, single encryption at gateway and double encryption support (gateway encryption with pass through of SSE headers to backend). If KMS is set up (either with Vault as KMS or using MINIO_SSE_MASTER_KEY),gateway will automatically perform single encryption. If MINIO_GATEWAY_SSE is set up in addition to Vault KMS, double encryption is performed.When neither KMS nor MINIO_GATEWAY_SSE is set, do a pass through to backend. When double encryption is specified, MINIO_GATEWAY_SSE can be set to "C" for SSE-C encryption at gateway and backend, "S3" for SSE-S3 encryption at gateway/backend or both to support more than one option. Fixes #6323, #6696
393 lines
11 KiB
Go
393 lines
11 KiB
Go
/*
|
|
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/minio/minio/pkg/hash"
|
|
)
|
|
|
|
// Initialize cache FS objects.
|
|
func initCacheFSObjects(disk string, cacheMaxUse int) (*cacheFSObjects, error) {
|
|
return newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
|
|
}
|
|
|
|
// inits diskCache struct for nDisks
|
|
func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
|
|
var cfs []*cacheFSObjects
|
|
for _, d := range drives {
|
|
obj, err := initCacheFSObjects(d, cacheMaxUse)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cfs = append(cfs, obj)
|
|
}
|
|
return &diskCache{cfs: cfs}, nil
|
|
}
|
|
|
|
// test whether a drive being offline causes
|
|
// getCacheFS to fetch next online drive
|
|
func TestGetCacheFS(t *testing.T) {
|
|
for n := 1; n < 10; n++ {
|
|
fsDirs, err := getRandomDisks(n)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d, err := initDiskCaches(fsDirs, 100, t)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bucketName := "testbucket"
|
|
objectName := "testobject"
|
|
ctx := context.Background()
|
|
// find cache drive where object would be hashed
|
|
index := d.hashIndex(bucketName, objectName)
|
|
// turn off drive by setting online status to false
|
|
d.cfs[index].online = false
|
|
cfs, err := d.getCacheFS(ctx, bucketName, objectName)
|
|
if n == 1 && err == errDiskNotFound {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
i := -1
|
|
for j, f := range d.cfs {
|
|
if f == cfs {
|
|
i = j
|
|
break
|
|
}
|
|
}
|
|
if i != (index+1)%n {
|
|
t.Fatalf("expected next cache location to be picked")
|
|
}
|
|
}
|
|
}
|
|
|
|
// test whether a drive being offline causes
|
|
// getCacheFS to fetch next online drive
|
|
func TestGetCacheFSMaxUse(t *testing.T) {
|
|
for n := 1; n < 10; n++ {
|
|
fsDirs, err := getRandomDisks(n)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bucketName := "testbucket"
|
|
objectName := "testobject"
|
|
ctx := context.Background()
|
|
// find cache drive where object would be hashed
|
|
index := d.hashIndex(bucketName, objectName)
|
|
// turn off drive by setting online status to false
|
|
d.cfs[index].online = false
|
|
cfs, err := d.getCacheFS(ctx, bucketName, objectName)
|
|
if n == 1 && err == errDiskNotFound {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
i := -1
|
|
for j, f := range d.cfs {
|
|
if f == cfs {
|
|
i = j
|
|
break
|
|
}
|
|
}
|
|
if i != (index+1)%n {
|
|
t.Fatalf("expected next cache location to be picked")
|
|
}
|
|
}
|
|
}
|
|
|
|
// test wildcard patterns for excluding entries from cache
|
|
func TestCacheExclusion(t *testing.T) {
|
|
fsDirs, err := getRandomDisks(1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cconfig := CacheConfig{Expiry: 30, Drives: fsDirs}
|
|
cobjects, err := newServerCacheObjects(cconfig)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cobj := cobjects.(*cacheObjects)
|
|
GlobalServiceDoneCh <- struct{}{}
|
|
testCases := []struct {
|
|
bucketName string
|
|
objectName string
|
|
excludePattern string
|
|
expectedResult bool
|
|
}{
|
|
{"testbucket", "testobjectmatch", "testbucket/testobj*", true},
|
|
{"testbucket", "testobjectnomatch", "testbucet/testobject*", false},
|
|
{"testbucket", "testobject/pref1/obj1", "*/*", true},
|
|
{"testbucket", "testobject/pref1/obj1", "*/pref1/*", true},
|
|
{"testbucket", "testobject/pref1/obj1", "testobject/*", false},
|
|
{"photos", "image1.jpg", "*.jpg", true},
|
|
{"photos", "europe/paris/seine.jpg", "seine.jpg", false},
|
|
{"photos", "europe/paris/seine.jpg", "*/seine.jpg", true},
|
|
{"phil", "z/likes/coffee", "*/likes/*", true},
|
|
{"failbucket", "no/slash/prefixes", "/failbucket/no/", false},
|
|
{"failbucket", "no/slash/prefixes", "/failbucket/no/*", false},
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
cobj.exclude = []string{testCase.excludePattern}
|
|
if cobj.isCacheExclude(testCase.bucketName, testCase.objectName) != testCase.expectedResult {
|
|
t.Fatal("Cache exclusion test failed for case ", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test diskCache.
|
|
func TestDiskCache(t *testing.T) {
|
|
fsDirs, err := getRandomDisks(1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d, err := initDiskCaches(fsDirs, 100, t)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cache := d.cfs[0]
|
|
ctx := context.Background()
|
|
bucketName := "testbucket"
|
|
objectName := "testobject"
|
|
content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
etag := "061208c10af71a30c6dcd6cf5d89f0fe"
|
|
contentType := "application/zip"
|
|
size := len(content)
|
|
|
|
httpMeta := make(map[string]string)
|
|
httpMeta["etag"] = etag
|
|
httpMeta["content-type"] = contentType
|
|
|
|
objInfo := ObjectInfo{}
|
|
objInfo.Bucket = bucketName
|
|
objInfo.Name = objectName
|
|
objInfo.Size = int64(size)
|
|
objInfo.ContentType = contentType
|
|
objInfo.ETag = etag
|
|
objInfo.UserDefined = httpMeta
|
|
opts := ObjectOptions{}
|
|
|
|
byteReader := bytes.NewReader([]byte(content))
|
|
hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !cache.Exists(ctx, bucketName, objectName) {
|
|
t.Fatal("Expected object to exist on cache")
|
|
}
|
|
if cachedObjInfo.ETag != objInfo.ETag {
|
|
t.Fatal("Expected ETag to match")
|
|
}
|
|
if cachedObjInfo.Size != objInfo.Size {
|
|
t.Fatal("Size mismatch")
|
|
}
|
|
if cachedObjInfo.ContentType != objInfo.ContentType {
|
|
t.Fatal("Cached content-type does not match")
|
|
}
|
|
writer := bytes.NewBuffer(nil)
|
|
err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
|
|
t.Errorf("wrong cached file content")
|
|
}
|
|
err = cache.Delete(ctx, bucketName, objectName)
|
|
if err != nil {
|
|
t.Errorf("object missing from cache")
|
|
}
|
|
online := cache.IsOnline()
|
|
if !online {
|
|
t.Errorf("expected cache drive to be online")
|
|
}
|
|
}
|
|
|
|
// Test diskCache with upper bound on max cache use.
|
|
func TestDiskCacheMaxUse(t *testing.T) {
|
|
fsDirs, err := getRandomDisks(1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cache := d.cfs[0]
|
|
ctx := context.Background()
|
|
bucketName := "testbucket"
|
|
objectName := "testobject"
|
|
content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
etag := "061208c10af71a30c6dcd6cf5d89f0fe"
|
|
contentType := "application/zip"
|
|
size := len(content)
|
|
|
|
httpMeta := make(map[string]string)
|
|
httpMeta["etag"] = etag
|
|
httpMeta["content-type"] = contentType
|
|
|
|
objInfo := ObjectInfo{}
|
|
objInfo.Bucket = bucketName
|
|
objInfo.Name = objectName
|
|
objInfo.Size = int64(size)
|
|
objInfo.ContentType = contentType
|
|
objInfo.ETag = etag
|
|
objInfo.UserDefined = httpMeta
|
|
opts := ObjectOptions{}
|
|
|
|
byteReader := bytes.NewReader([]byte(content))
|
|
hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !cache.diskAvailable(int64(size)) {
|
|
err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
|
|
if err != errDiskFull {
|
|
t.Fatal("Cache max-use limit violated.")
|
|
}
|
|
} else {
|
|
err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !cache.Exists(ctx, bucketName, objectName) {
|
|
t.Fatal("Expected object to exist on cache")
|
|
}
|
|
if cachedObjInfo.ETag != objInfo.ETag {
|
|
t.Fatal("Expected ETag to match")
|
|
}
|
|
if cachedObjInfo.Size != objInfo.Size {
|
|
t.Fatal("Size mismatch")
|
|
}
|
|
if cachedObjInfo.ContentType != objInfo.ContentType {
|
|
t.Fatal("Cached content-type does not match")
|
|
}
|
|
writer := bytes.NewBuffer(nil)
|
|
err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
|
|
t.Errorf("wrong cached file content")
|
|
}
|
|
err = cache.Delete(ctx, bucketName, objectName)
|
|
if err != nil {
|
|
t.Errorf("object missing from cache")
|
|
}
|
|
online := cache.IsOnline()
|
|
if !online {
|
|
t.Errorf("expected cache drive to be online")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIsCacheExcludeDirective(t *testing.T) {
|
|
testCases := []struct {
|
|
cacheControlOpt string
|
|
expectedResult bool
|
|
}{
|
|
{"no-cache", true},
|
|
{"no-store", true},
|
|
{"must-revalidate", true},
|
|
{"no-transform", false},
|
|
{"max-age", false},
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
if isCacheExcludeDirective(testCase.cacheControlOpt) != testCase.expectedResult {
|
|
t.Errorf("Cache exclude directive test failed for case %d", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetCacheControlOpts(t *testing.T) {
|
|
testCases := []struct {
|
|
cacheControlHeaderVal string
|
|
expiryHeaderVal string
|
|
expectedCacheControl cacheControl
|
|
expectedErr bool
|
|
}{
|
|
{"", "", cacheControl{}, false},
|
|
{"max-age=2592000, public", "", cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: false}, false},
|
|
{"max-age=2592000, no-store", "", cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: true}, false},
|
|
{"must-revalidate, max-age=600", "", cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: true}, false},
|
|
{"s-maxAge=2500, max-age=600", "", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}, exclude: false}, false},
|
|
{"s-maxAge=2500, max-age=600", "Wed, 21 Oct 2015 07:28:00 GMT", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC), exclude: false}, false},
|
|
{"s-maxAge=2500, max-age=600s", "", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}, exclude: false}, true},
|
|
}
|
|
var m map[string]string
|
|
|
|
for i, testCase := range testCases {
|
|
m = make(map[string]string)
|
|
m["cache-control"] = testCase.cacheControlHeaderVal
|
|
if testCase.expiryHeaderVal != "" {
|
|
m["expires"] = testCase.expiryHeaderVal
|
|
}
|
|
c, err := getCacheControlOpts(m)
|
|
if testCase.expectedErr && err == nil {
|
|
t.Errorf("expected err for case %d", i)
|
|
}
|
|
if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
|
|
t.Errorf("expected %v got %v for case %d", testCase.expectedCacheControl, c, i)
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
func TestFilterFromCache(t *testing.T) {
|
|
testCases := []struct {
|
|
metadata map[string]string
|
|
expectedResult bool
|
|
}{
|
|
{map[string]string{"content-type": "application/json"}, false},
|
|
{map[string]string{"cache-control": "private,no-store"}, true},
|
|
{map[string]string{"cache-control": "no-cache,must-revalidate"}, true},
|
|
{map[string]string{"cache-control": "no-transform"}, false},
|
|
{map[string]string{"cache-control": "max-age=3600"}, false},
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
if filterFromCache(testCase.metadata) != testCase.expectedResult {
|
|
t.Errorf("Cache exclude directive test failed for case %d", i)
|
|
}
|
|
}
|
|
}
|