mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
Add large bucket support for erasure coded backend (#5160)
This PR implements an object layer which combines input erasure sets of XL layers into a unified namespace. This object layer extends the existing erasure coded implementation, it is assumed in this design that providing > 16 disks is a static configuration as well i.e if you started the setup with 32 disks with 4 sets 8 disks per pack then you would need to provide 4 sets always. Some design details and restrictions: - Objects are distributed using consistent ordering to a unique erasure coded layer. - Each pack has its own dsync so locks are synchronized properly at pack (erasure layer). - Each pack still has a maximum of 16 disks requirement, you can start with multiple such sets statically. - Static sets set of disks and cannot be changed, there is no elastic expansion allowed. - Static sets set of disks and cannot be changed, there is no elastic removal allowed. - ListObjects() across sets can be noticeably slower since List happens on all servers, and is merged at this sets layer. Fixes #5465 Fixes #5464 Fixes #5461 Fixes #5460 Fixes #5459 Fixes #5458 Fixes #5460 Fixes #5488 Fixes #5489 Fixes #5497 Fixes #5496
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
dd80256151
commit
fb96779a8a
388
cmd/endpoint-ellipses_test.go
Normal file
388
cmd/endpoint-ellipses_test.go
Normal file
@@ -0,0 +1,388 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/ellipses"
|
||||
)
|
||||
|
||||
// Tests create endpoints with ellipses and without.
|
||||
func TestCreateServerEndpoints(t *testing.T) {
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
args []string
|
||||
success bool
|
||||
}{
|
||||
// Invalid input.
|
||||
{"", []string{}, false},
|
||||
// Range cannot be negative.
|
||||
{":9000", []string{"/export1{-1...1}"}, false},
|
||||
// Range cannot start bigger than end.
|
||||
{":9000", []string{"/export1{64...1}"}, false},
|
||||
// Range can only be numeric.
|
||||
{":9000", []string{"/export1{a...z}"}, false},
|
||||
// Duplicate disks not allowed.
|
||||
{":9000", []string{"/export1{1...32}", "/export1{1...32}"}, false},
|
||||
// Same host cannot export same disk on two ports - special case localhost.
|
||||
{":9001", []string{"http://localhost:900{1...2}/export{1...64}"}, false},
|
||||
|
||||
// Valid inputs.
|
||||
{":9000", []string{"/export1"}, true},
|
||||
{":9000", []string{"/export1", "/export2", "/export3", "/export4"}, true},
|
||||
{":9000", []string{"/export1{1...64}"}, true},
|
||||
{":9000", []string{"/export1{01...64}"}, true},
|
||||
{":9000", []string{"/export1{1...32}", "/export1{33...64}"}, true},
|
||||
{":9001", []string{"http://localhost:9001/export{1...64}"}, true},
|
||||
{":9001", []string{"http://localhost:9001/export{01...64}"}, true},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
_, _, _, _, _, err := createServerEndpoints(testCase.serverAddr, testCase.args...)
|
||||
if err != nil && testCase.success {
|
||||
t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
|
||||
}
|
||||
if err == nil && !testCase.success {
|
||||
t.Errorf("Test %d: Expected failure but passed instead", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test tests calculating set indexes.
|
||||
func TestGetSetIndexes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
args []string
|
||||
totalSizes []uint64
|
||||
indexes [][]uint64
|
||||
success bool
|
||||
}{
|
||||
// Invalid inputs.
|
||||
{
|
||||
[]string{"data{1...27}"},
|
||||
[]uint64{27},
|
||||
nil,
|
||||
false,
|
||||
},
|
||||
// Valid inputs.
|
||||
{
|
||||
[]string{"data{1...64}"},
|
||||
[]uint64{64},
|
||||
[][]uint64{[]uint64{16, 16, 16, 16}},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"data{1...24}"},
|
||||
[]uint64{24},
|
||||
[][]uint64{[]uint64{12, 12}},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"data/controller{1...11}/export{1...8}"},
|
||||
[]uint64{88},
|
||||
[][]uint64{[]uint64{8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8}},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"data{1...4}"},
|
||||
[]uint64{4},
|
||||
[][]uint64{[]uint64{4}},
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
|
||||
gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes)
|
||||
if err != nil && testCase.success {
|
||||
t.Errorf("Expected success but failed instead %s", err)
|
||||
}
|
||||
if err == nil && !testCase.success {
|
||||
t.Errorf("Expected failure but passed instead")
|
||||
}
|
||||
if !reflect.DeepEqual(testCase.indexes, gotIndexes) {
|
||||
t.Errorf("Expected %v, got %v", testCase.indexes, gotIndexes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func getSequences(start int, number int, paddinglen int) (seq []string) {
|
||||
for i := start; i <= number; i++ {
|
||||
if paddinglen == 0 {
|
||||
seq = append(seq, fmt.Sprintf("%d", i))
|
||||
} else {
|
||||
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dd", paddinglen), i))
|
||||
}
|
||||
}
|
||||
return seq
|
||||
}
|
||||
|
||||
// Test tests parses endpoint ellipses input pattern.
|
||||
func TestParseEndpointSet(t *testing.T) {
|
||||
testCases := []struct {
|
||||
arg string
|
||||
es endpointSet
|
||||
success bool
|
||||
}{
|
||||
// Tests invalid inputs.
|
||||
{
|
||||
"...",
|
||||
endpointSet{},
|
||||
false,
|
||||
},
|
||||
// Indivisible range.
|
||||
{
|
||||
"{1...27}",
|
||||
endpointSet{},
|
||||
false,
|
||||
},
|
||||
// No range specified.
|
||||
{
|
||||
"{...}",
|
||||
endpointSet{},
|
||||
false,
|
||||
},
|
||||
// Invalid range.
|
||||
{
|
||||
"http://minio{2...3}/export/set{1...0}",
|
||||
endpointSet{},
|
||||
false,
|
||||
},
|
||||
// Range cannot be smaller than 4 minimum.
|
||||
{
|
||||
"/export{1..2}",
|
||||
endpointSet{},
|
||||
false,
|
||||
},
|
||||
// Unsupported characters.
|
||||
{
|
||||
"/export/test{1...2O}",
|
||||
endpointSet{},
|
||||
false,
|
||||
},
|
||||
// Tests valid inputs.
|
||||
{
|
||||
"/export/set{1...64}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"/export/set",
|
||||
"",
|
||||
getSequences(1, 64, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{16, 16, 16, 16}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// Valid input for distributed setup.
|
||||
{
|
||||
"http://minio{2...3}/export/set{1...64}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"",
|
||||
getSequences(1, 64, 0),
|
||||
},
|
||||
{
|
||||
"http://minio",
|
||||
"/export/set",
|
||||
getSequences(2, 3, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{16, 16, 16, 16, 16, 16, 16, 16}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// Supporting some advanced cases.
|
||||
{
|
||||
"http://minio{1...64}.mydomain.net/data",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"http://minio",
|
||||
".mydomain.net/data",
|
||||
getSequences(1, 64, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{16, 16, 16, 16}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"http://rack{1...4}.mydomain.minio{1...16}/data",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"/data",
|
||||
getSequences(1, 16, 0),
|
||||
},
|
||||
{
|
||||
"http://rack",
|
||||
".mydomain.minio",
|
||||
getSequences(1, 4, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{16, 16, 16, 16}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// Supporting kubernetes cases.
|
||||
{
|
||||
"http://minio{0...15}.mydomain.net/data{0...1}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"",
|
||||
getSequences(0, 1, 0),
|
||||
},
|
||||
{
|
||||
"http://minio",
|
||||
".mydomain.net/data",
|
||||
getSequences(0, 15, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{16, 16}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// No host regex, just disks.
|
||||
{
|
||||
"http://server1/data{1...32}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"http://server1/data",
|
||||
"",
|
||||
getSequences(1, 32, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{16, 16}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// No host regex, just disks with two position numerics.
|
||||
{
|
||||
"http://server1/data{01...32}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"http://server1/data",
|
||||
"",
|
||||
getSequences(1, 32, 2),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{16, 16}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// More than 2 ellipses are supported as well.
|
||||
{
|
||||
"http://minio{2...3}/export/set{1...64}/test{1...2}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"",
|
||||
getSequences(1, 2, 0),
|
||||
},
|
||||
{
|
||||
"",
|
||||
"/test",
|
||||
getSequences(1, 64, 0),
|
||||
},
|
||||
{
|
||||
"http://minio",
|
||||
"/export/set",
|
||||
getSequences(2, 3, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{16, 16, 16, 16, 16, 16, 16, 16,
|
||||
16, 16, 16, 16, 16, 16, 16, 16}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// More than 1 ellipses per argument for standalone setup.
|
||||
{
|
||||
"/export{1...10}/disk{1...10}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"",
|
||||
getSequences(1, 10, 0),
|
||||
},
|
||||
{
|
||||
"/export",
|
||||
"/disk",
|
||||
getSequences(1, 10, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{[]uint64{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
|
||||
gotEs, err := parseEndpointSet(testCase.arg)
|
||||
if err != nil && testCase.success {
|
||||
t.Errorf("Expected success but failed instead %s", err)
|
||||
}
|
||||
if err == nil && !testCase.success {
|
||||
t.Errorf("Expected failure but passed instead")
|
||||
}
|
||||
if !reflect.DeepEqual(testCase.es, gotEs) {
|
||||
t.Errorf("Expected %v, got %v", testCase.es, gotEs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user