MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 21:49:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix storage class related issues (#5338)

Browse Source 
- Update startup banner to print storage class in capitals. This
makes it easier to identify different storage classes available.

- Update response metadata to not send STANDARD storage class.
This is in accordance with AWS S3 behaviour.

- Update minio-go library to bring in storage class related
changes. This is needed to make transparent translation of
storage class headers for Minio S3 Gateway.
This commit is contained in:
Nitish Tiwari
2018-01-04 11:44:45 +05:30
committed by GitHub
parent 6f7c6fc560
commit 1e5fb4b79a
18 changed files with 7334 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/fs-v1-metadata.go
View File

@@ -122,7 +122,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
// etag/md5Sum has already been extracted. We need to
// remove to avoid it from appearing as part of
// response headers. e.g, X-Minio-* or X-Amz-*.
objInfo.UserDefined = cleanMetaETag(m.Meta)
objInfo.UserDefined = cleanMetadata(m.Meta)
// Success..
return objInfo

24
cmd/object-api-utils.go
View File

@@ -187,14 +187,26 @@ func getCompleteMultipartMD5(parts []CompletePart) (string, error) {
return s3MD5, nil
}
// Clean meta etag keys 'md5Sum', 'etag'.
func cleanMetaETag(metadata map[string]string) map[string]string {
return cleanMetadata(metadata, "md5Sum", "etag")
// Clean unwanted fields from metadata
func cleanMetadata(metadata map[string]string) map[string]string {
// Remove STANDARD StorageClass
metadata = removeStandardStorageClass(metadata)
// Clean meta etag keys 'md5Sum', 'etag'.
return cleanMetadataKeys(metadata, "md5Sum", "etag")
}
// Clean metadata takes keys to be filtered
// and returns a new map with the keys filtered.
func cleanMetadata(metadata map[string]string, keyNames ...string) map[string]string {
// Filter X-Amz-Storage-Class field only if it is set to STANDARD.
// This is done since AWS S3 doesn't return STANDARD Storage class as response header.
func removeStandardStorageClass(metadata map[string]string) map[string]string {
if metadata[amzStorageClass] == standardStorageClass {
delete(metadata, amzStorageClass)
}
return metadata
}
// cleanMetadataKeys takes keyNames to be filtered
// and returns a new map with all the entries with keyNames removed.
func cleanMetadataKeys(metadata map[string]string, keyNames ...string) map[string]string {
var newMeta = make(map[string]string)
for k, v := range metadata {
if contains(keyNames, k) {

98
cmd/object-api-utils_test.go
View File

@@ -17,6 +17,7 @@
package cmd
import (
"reflect"
"testing"
)
@@ -197,3 +198,100 @@ func TestIsMinioMetaBucketName(t *testing.T) {
}
}
}
// Tests RemoveStandardStorageClass method. Expectation is metadata map
// should be cleared of x-amz-storage-class, if it is set to STANDARD
func TestRemoveStandardStorageClass(t *testing.T) {
tests := []struct {
name string
metadata map[string]string
want map[string]string
}{
{
name: "1",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"},
want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
},
{
name: "2",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
},
{
name: "3",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
},
}
for _, tt := range tests {
if got := removeStandardStorageClass(tt.metadata); !reflect.DeepEqual(got, tt.want) {
t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
}
}
}
// Tests CleanMetadata method. Expectation is metadata map
// should be cleared of etag, md5Sum and x-amz-storage-class, if it is set to STANDARD
func TestCleanMetadata(t *testing.T) {
tests := []struct {
name string
metadata map[string]string
want map[string]string
}{
{
name: "1",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"},
want: map[string]string{"content-type": "application/octet-stream"},
},
{
name: "2",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
},
{
name: "3",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "md5Sum": "abcde"},
want: map[string]string{"content-type": "application/octet-stream"},
},
}
for _, tt := range tests {
if got := cleanMetadata(tt.metadata); !reflect.DeepEqual(got, tt.want) {
t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
}
}
}
// Tests CleanMetadataKeys method. Expectation is metadata map
// should be cleared of keys passed to CleanMetadataKeys method
func TestCleanMetadataKeys(t *testing.T) {
tests := []struct {
name string
metadata map[string]string
keys []string
want map[string]string
}{
{
name: "1",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD", "md5": "abcde"},
keys: []string{"etag", "md5"},
want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "STANDARD"},
},
{
name: "2",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY", "md5sum": "abcde"},
keys: []string{"etag", "md5sum"},
want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
},
{
name: "3",
metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "xyz": "abcde"},
keys: []string{"etag", "xyz"},
want: map[string]string{"content-type": "application/octet-stream"},
},
}
for _, tt := range tests {
if got := cleanMetadataKeys(tt.metadata, tt.keys...); !reflect.DeepEqual(got, tt.want) {
t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
}
}
}

4
cmd/server-startup-msg.go
View File

@@ -198,7 +198,7 @@ func printStorageClassInfoMsg(storageInfo StorageInfo) {
func getStandardStorageClassInfoMsg(storageInfo StorageInfo) string {
var msg string
if maxDiskFailures := storageInfo.Backend.standardSCParity - storageInfo.Backend.OfflineDisks; maxDiskFailures >= 0 {
msg += fmt.Sprintf("Objects with Standard class can withstand [%d] drive failure(s).\n", maxDiskFailures)
msg += fmt.Sprintf("Objects with "+standardStorageClass+" class can withstand [%d] drive failure(s).\n", maxDiskFailures)
}
return msg
}
@@ -206,7 +206,7 @@ func getStandardStorageClassInfoMsg(storageInfo StorageInfo) string {
func getRRSStorageClassInfoMsg(storageInfo StorageInfo) string {
var msg string
if maxDiskFailures := storageInfo.Backend.rrSCParity - storageInfo.Backend.OfflineDisks; maxDiskFailures >= 0 {
msg += fmt.Sprintf("Objects with Reduced Redundancy class can withstand [%d] drive failure(s).\n", maxDiskFailures)
msg += fmt.Sprintf("Objects with "+reducedRedundancyStorageClass+" class can withstand [%d] drive failure(s).\n", maxDiskFailures)
}
return msg
}

12
cmd/server-startup-msg_test.go
View File

@@ -172,7 +172,7 @@ func TestGetStandardStorageClassInfoMsg(t *testing.T) {
standardSCParity int
rrSCParity int
}{Erasure, 15, 1, 5, 3},
}, "Objects with Standard class can withstand [4] drive failure(s).\n"},
}, "Objects with " + standardStorageClass + " class can withstand [4] drive failure(s).\n"},
{"2", StorageInfo{
Total: 30 * humanize.GiByte,
Free: 3 * humanize.GiByte,
@@ -183,7 +183,7 @@ func TestGetStandardStorageClassInfoMsg(t *testing.T) {
standardSCParity int
rrSCParity int
}{Erasure, 10, 0, 5, 3},
}, "Objects with Standard class can withstand [5] drive failure(s).\n"},
}, "Objects with " + standardStorageClass + " class can withstand [5] drive failure(s).\n"},
{"3", StorageInfo{
Total: 15 * humanize.GiByte,
Free: 2 * humanize.GiByte,
@@ -194,7 +194,7 @@ func TestGetStandardStorageClassInfoMsg(t *testing.T) {
standardSCParity int
rrSCParity int
}{Erasure, 12, 3, 6, 2},
}, "Objects with Standard class can withstand [3] drive failure(s).\n"},
}, "Objects with " + standardStorageClass + " class can withstand [3] drive failure(s).\n"},
}
for _, tt := range tests {
if got := getStandardStorageClassInfoMsg(tt.args); got != tt.want {
@@ -219,7 +219,7 @@ func TestGetRRSStorageClassInfoMsg(t *testing.T) {
standardSCParity int
rrSCParity int
}{Erasure, 15, 1, 5, 3},
}, "Objects with Reduced Redundancy class can withstand [2] drive failure(s).\n"},
}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [2] drive failure(s).\n"},
{"2", StorageInfo{
Total: 30 * humanize.GiByte,
Free: 3 * humanize.GiByte,
@@ -230,7 +230,7 @@ func TestGetRRSStorageClassInfoMsg(t *testing.T) {
standardSCParity int
rrSCParity int
}{Erasure, 16, 0, 5, 3},
}, "Objects with Reduced Redundancy class can withstand [3] drive failure(s).\n"},
}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [3] drive failure(s).\n"},
{"3", StorageInfo{
Total: 15 * humanize.GiByte,
Free: 2 * humanize.GiByte,
@@ -241,7 +241,7 @@ func TestGetRRSStorageClassInfoMsg(t *testing.T) {
standardSCParity int
rrSCParity int
}{Erasure, 12, 3, 6, 5},
}, "Objects with Reduced Redundancy class can withstand [2] drive failure(s).\n"},
}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [2] drive failure(s).\n"},
}
for _, tt := range tests {
if got := getRRSStorageClassInfoMsg(tt.args); got != tt.want {

2
cmd/xl-v1-metadata.go
View File

@@ -296,7 +296,7 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo {
// etag/md5Sum has already been extracted. We need to
// remove to avoid it from appearing as part of
// response headers. e.g, X-Minio-* or X-Amz-*.
objInfo.UserDefined = cleanMetaETag(m.Meta)
objInfo.UserDefined = cleanMetadata(m.Meta)
// Success.
return objInfo

2
cmd/xl-v1-object.go
View File

@@ -347,7 +347,7 @@ func (xl xlObjects) getObjectInfo(bucket, object string) (objInfo ObjectInfo, er
// etag/md5Sum has already been extracted. We need to
// remove to avoid it from appearing as part of
// response headers. e.g, X-Minio-* or X-Amz-*.
objInfo.UserDefined = cleanMetaETag(xlMetaMap)
objInfo.UserDefined = cleanMetadata(xlMetaMap)
// Success.
return objInfo, nil