mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
Move storageclass config handling into cmd/config/storageclass (#8360)
Continuation of the changes done in PR #8351 to refactor, add tests and move global handling into a more idiomatic style for Go as packages.
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
002ac82631
commit
3b8adf7528
230
cmd/config/storageclass/storage-class.go
Normal file
230
cmd/config/storageclass/storage-class.go
Normal file
@@ -0,0 +1,230 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package storageclass
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
)
|
||||
|
||||
// Standard constants for all storage class
|
||||
const (
|
||||
// Reduced redundancy storage class
|
||||
RRS = "REDUCED_REDUNDANCY"
|
||||
// Standard storage class
|
||||
STANDARD = "STANDARD"
|
||||
)
|
||||
|
||||
// Standard constats for config info storage class
|
||||
const (
|
||||
// Reduced redundancy storage class environment variable
|
||||
RRSEnv = "MINIO_STORAGE_CLASS_RRS"
|
||||
// Standard storage class environment variable
|
||||
StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
|
||||
|
||||
// Supported storage class scheme is EC
|
||||
schemePrefix = "EC"
|
||||
|
||||
// Min parity disks
|
||||
minParityDisks = 2
|
||||
|
||||
// Default RRS parity is always minimum parity.
|
||||
defaultRRSParity = minParityDisks
|
||||
)
|
||||
|
||||
// StorageClass - holds storage class information
|
||||
type StorageClass struct {
|
||||
Parity int
|
||||
}
|
||||
|
||||
// Config storage class configuration
|
||||
type Config struct {
|
||||
Standard StorageClass `json:"standard"`
|
||||
RRS StorageClass `json:"rrs"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
|
||||
func (sCfg *Config) UnmarshalJSON(data []byte) error {
|
||||
type Alias Config
|
||||
aux := &struct {
|
||||
*Alias
|
||||
}{
|
||||
Alias: (*Alias)(sCfg),
|
||||
}
|
||||
return json.Unmarshal(data, &aux)
|
||||
}
|
||||
|
||||
// IsValid - returns true if input string is a valid
|
||||
// storage class kind supported.
|
||||
func IsValid(sc string) bool {
|
||||
return sc == RRS || sc == STANDARD
|
||||
}
|
||||
|
||||
// UnmarshalText unmarshals storage class from its textual form into
|
||||
// storageClass structure.
|
||||
func (sc *StorageClass) UnmarshalText(b []byte) error {
|
||||
scStr := string(b)
|
||||
if scStr == "" {
|
||||
return nil
|
||||
}
|
||||
s, err := parseStorageClass(scStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sc.Parity = s.Parity
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalText - marshals storage class string.
|
||||
func (sc *StorageClass) MarshalText() ([]byte, error) {
|
||||
if sc.Parity != 0 {
|
||||
return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
|
||||
}
|
||||
return []byte(""), nil
|
||||
}
|
||||
|
||||
func (sc *StorageClass) String() string {
|
||||
if sc.Parity != 0 {
|
||||
return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Parses given storageClassEnv and returns a storageClass structure.
|
||||
// Supported Storage Class format is "Scheme:Number of parity disks".
|
||||
// Currently only supported scheme is "EC".
|
||||
func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
|
||||
s := strings.Split(storageClassEnv, ":")
|
||||
|
||||
// only two elements allowed in the string - "scheme" and "number of parity disks"
|
||||
if len(s) > 2 {
|
||||
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv)
|
||||
} else if len(s) < 2 {
|
||||
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv)
|
||||
}
|
||||
|
||||
// only allowed scheme is "EC"
|
||||
if s[0] != schemePrefix {
|
||||
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC")
|
||||
}
|
||||
|
||||
// Number of parity disks should be integer
|
||||
parityDisks, err := strconv.Atoi(s[1])
|
||||
if err != nil {
|
||||
return StorageClass{}, config.ErrStorageClassValue(err)
|
||||
}
|
||||
|
||||
return StorageClass{
|
||||
Parity: parityDisks,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Validates the parity disks.
|
||||
func validateParity(ssParity, rrsParity, drivesPerSet int) (err error) {
|
||||
if ssParity == 0 && rrsParity == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SS parity disks should be greater than or equal to minParityDisks.
|
||||
// Parity below minParityDisks is not supported.
|
||||
if ssParity < minParityDisks {
|
||||
return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
|
||||
ssParity, minParityDisks)
|
||||
}
|
||||
|
||||
// RRS parity disks should be greater than or equal to minParityDisks.
|
||||
// Parity below minParityDisks is not supported.
|
||||
if rrsParity < minParityDisks {
|
||||
return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDisks)
|
||||
}
|
||||
|
||||
if ssParity > drivesPerSet/2 {
|
||||
return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, drivesPerSet/2)
|
||||
}
|
||||
|
||||
if rrsParity > drivesPerSet/2 {
|
||||
return fmt.Errorf("Reduced redundancy storage class parity %d should be less than or equal to %d", rrsParity, drivesPerSet/2)
|
||||
}
|
||||
|
||||
if ssParity > 0 && rrsParity > 0 {
|
||||
if ssParity < rrsParity {
|
||||
return fmt.Errorf("Standard storage class parity disks %d should be greater than or equal to Reduced redundancy storage class parity disks %d", ssParity, rrsParity)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetParityForSC - Returns the data and parity drive count based on storage class
|
||||
// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and MINIO_STORAGE_CLASS_STANDARD
|
||||
// or config.json fields
|
||||
// -- corresponding values are returned
|
||||
// If storage class is not set during startup, default values are returned
|
||||
// -- Default for Reduced Redundancy Storage class is, parity = 2 and data = N-Parity
|
||||
// -- Default for Standard Storage class is, parity = N/2, data = N/2
|
||||
// If storage class is empty
|
||||
// -- standard storage class is assumed and corresponding data and parity is returned
|
||||
func (sCfg Config) GetParityForSC(sc string) (parity int) {
|
||||
switch strings.TrimSpace(sc) {
|
||||
case RRS:
|
||||
// set the rrs parity if available
|
||||
if sCfg.RRS.Parity == 0 {
|
||||
return defaultRRSParity
|
||||
}
|
||||
return sCfg.RRS.Parity
|
||||
default:
|
||||
return sCfg.Standard.Parity
|
||||
}
|
||||
}
|
||||
|
||||
// LookupConfig - lookup storage class config and override with valid environment settings if any.
|
||||
func LookupConfig(cfg Config, drivesPerSet int) (Config, error) {
|
||||
var err error
|
||||
|
||||
// Check for environment variables and parse into storageClass struct
|
||||
if ssc := env.Get(StandardEnv, cfg.Standard.String()); ssc != "" {
|
||||
cfg.Standard, err = parseStorageClass(ssc)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
if cfg.Standard.Parity == 0 {
|
||||
cfg.Standard.Parity = drivesPerSet / 2
|
||||
}
|
||||
}
|
||||
|
||||
if rrsc := env.Get(RRSEnv, cfg.RRS.String()); rrsc != "" {
|
||||
cfg.RRS, err = parseStorageClass(rrsc)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
if cfg.RRS.Parity == 0 {
|
||||
cfg.RRS.Parity = defaultRRSParity
|
||||
}
|
||||
}
|
||||
|
||||
// Validation is done after parsing both the storage classes. This is needed because we need one
|
||||
// storage class value to deduce the correct value of the other storage class.
|
||||
if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, drivesPerSet); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
163
cmd/config/storageclass/storage-class_test.go
Normal file
163
cmd/config/storageclass/storage-class_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package storageclass
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseStorageClass(t *testing.T) {
|
||||
tests := []struct {
|
||||
storageClassEnv string
|
||||
wantSc StorageClass
|
||||
expectedError error
|
||||
}{
|
||||
{"EC:3", StorageClass{
|
||||
Parity: 3},
|
||||
nil},
|
||||
{"EC:4", StorageClass{
|
||||
Parity: 4},
|
||||
nil},
|
||||
{"AB:4", StorageClass{
|
||||
Parity: 4},
|
||||
errors.New("Unsupported scheme AB. Supported scheme is EC")},
|
||||
{"EC:4:5", StorageClass{
|
||||
Parity: 4},
|
||||
errors.New("Too many sections in EC:4:5")},
|
||||
{"EC:A", StorageClass{
|
||||
Parity: 4},
|
||||
errors.New(`strconv.Atoi: parsing "A": invalid syntax`)},
|
||||
{"AB", StorageClass{
|
||||
Parity: 4},
|
||||
errors.New("Too few sections in AB")},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
gotSc, err := parseStorageClass(tt.storageClassEnv)
|
||||
if err != nil && tt.expectedError == nil {
|
||||
t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
|
||||
return
|
||||
}
|
||||
if err == nil && tt.expectedError != nil {
|
||||
t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
|
||||
return
|
||||
}
|
||||
if tt.expectedError == nil && !reflect.DeepEqual(gotSc, tt.wantSc) {
|
||||
t.Errorf("Test %d, Expected %v, got %v", i+1, tt.wantSc, gotSc)
|
||||
return
|
||||
}
|
||||
if tt.expectedError != nil && err.Error() != tt.expectedError.Error() {
|
||||
t.Errorf("Test %d, Expected `%v`, got `%v`", i+1, tt.expectedError, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateParity(t *testing.T) {
|
||||
tests := []struct {
|
||||
rrsParity int
|
||||
ssParity int
|
||||
success bool
|
||||
drivesPerSet int
|
||||
}{
|
||||
{2, 4, true, 16},
|
||||
{3, 3, true, 16},
|
||||
{0, 0, true, 16},
|
||||
{1, 4, false, 16},
|
||||
{7, 6, false, 16},
|
||||
{9, 0, false, 16},
|
||||
{9, 9, false, 16},
|
||||
{2, 9, false, 16},
|
||||
{9, 2, false, 16},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
err := validateParity(tt.ssParity, tt.rrsParity, tt.drivesPerSet)
|
||||
if err != nil && tt.success {
|
||||
t.Errorf("Test %d, Expected success, got %s", i+1, err)
|
||||
}
|
||||
if err == nil && !tt.success {
|
||||
t.Errorf("Test %d, Expected failure, got success", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParityCount(t *testing.T) {
|
||||
tests := []struct {
|
||||
sc string
|
||||
disksCount int
|
||||
expectedData int
|
||||
expectedParity int
|
||||
}{
|
||||
{RRS, 16, 14, 2},
|
||||
{STANDARD, 16, 8, 8},
|
||||
{"", 16, 8, 8},
|
||||
{RRS, 16, 9, 7},
|
||||
{STANDARD, 16, 10, 6},
|
||||
{"", 16, 9, 7},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
scfg := Config{
|
||||
Standard: StorageClass{
|
||||
Parity: 8,
|
||||
},
|
||||
RRS: StorageClass{
|
||||
Parity: 0,
|
||||
},
|
||||
}
|
||||
// Set env var for test case 4
|
||||
if i+1 == 4 {
|
||||
scfg.RRS.Parity = 7
|
||||
}
|
||||
// Set env var for test case 5
|
||||
if i+1 == 5 {
|
||||
scfg.Standard.Parity = 6
|
||||
}
|
||||
// Set env var for test case 6
|
||||
if i+1 == 6 {
|
||||
scfg.Standard.Parity = 7
|
||||
}
|
||||
parity := scfg.GetParityForSC(tt.sc)
|
||||
if (tt.disksCount - parity) != tt.expectedData {
|
||||
t.Errorf("Test %d, Expected data disks %d, got %d", i+1, tt.expectedData, tt.disksCount-parity)
|
||||
continue
|
||||
}
|
||||
if parity != tt.expectedParity {
|
||||
t.Errorf("Test %d, Expected parity disks %d, got %d", i+1, tt.expectedParity, parity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test IsValid method with valid and invalid inputs
|
||||
func TestIsValidStorageClassKind(t *testing.T) {
|
||||
tests := []struct {
|
||||
sc string
|
||||
want bool
|
||||
}{
|
||||
{"STANDARD", true},
|
||||
{"REDUCED_REDUNDANCY", true},
|
||||
{"", false},
|
||||
{"INVALID", false},
|
||||
{"123", false},
|
||||
{"MINIO_STORAGE_CLASS_RRS", false},
|
||||
{"MINIO_STORAGE_CLASS_STANDARD", false},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
if got := IsValid(tt.sc); got != tt.want {
|
||||
t.Errorf("Test %d, Expected Storage Class to be %t, got %t", i+1, tt.want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user