mirror of
https://github.com/minio/minio.git
synced 2025-11-09 13:39:46 -05:00
Add large bucket support for erasure coded backend (#5160)
This PR implements an object layer which combines input erasure sets of XL layers into a unified namespace. This object layer extends the existing erasure coded implementation, it is assumed in this design that providing > 16 disks is a static configuration as well i.e if you started the setup with 32 disks with 4 sets 8 disks per pack then you would need to provide 4 sets always. Some design details and restrictions: - Objects are distributed using consistent ordering to a unique erasure coded layer. - Each pack has its own dsync so locks are synchronized properly at pack (erasure layer). - Each pack still has a maximum of 16 disks requirement, you can start with multiple such sets statically. - Static sets set of disks and cannot be changed, there is no elastic expansion allowed. - Static sets set of disks and cannot be changed, there is no elastic removal allowed. - ListObjects() across sets can be noticeably slower since List happens on all servers, and is merged at this sets layer. Fixes #5465 Fixes #5464 Fixes #5461 Fixes #5460 Fixes #5459 Fixes #5458 Fixes #5460 Fixes #5488 Fixes #5489 Fixes #5497 Fixes #5496
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
dd80256151
commit
fb96779a8a
207
pkg/ellipses/ellipses.go
Normal file
207
pkg/ellipses/ellipses.go
Normal file
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package ellipses
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
// Regex to extract ellipses syntax inputs.
|
||||
regexpEllipses = regexp.MustCompile(`(.*)({[0-9]*\.\.\.[0-9]*})(.*)`)
|
||||
|
||||
// Ellipses constants
|
||||
openBraces = "{"
|
||||
closeBraces = "}"
|
||||
ellipses = "..."
|
||||
)
|
||||
|
||||
// Parses an ellipses range pattern of following style
|
||||
// `{1...64}`
|
||||
// `{33...64}`
|
||||
func parseEllipsesRange(pattern string) (seq []string, err error) {
|
||||
if strings.Index(pattern, openBraces) == -1 {
|
||||
return nil, errors.New("Invalid argument")
|
||||
}
|
||||
if strings.Index(pattern, closeBraces) == -1 {
|
||||
return nil, errors.New("Invalid argument")
|
||||
}
|
||||
|
||||
pattern = strings.TrimPrefix(pattern, openBraces)
|
||||
pattern = strings.TrimSuffix(pattern, closeBraces)
|
||||
|
||||
ellipsesRange := strings.Split(pattern, ellipses)
|
||||
if len(ellipsesRange) != 2 {
|
||||
return nil, errors.New("Invalid argument")
|
||||
}
|
||||
|
||||
var start, end uint64
|
||||
if start, err = strconv.ParseUint(ellipsesRange[0], 10, 64); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if end, err = strconv.ParseUint(ellipsesRange[1], 10, 64); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if start > end {
|
||||
return nil, fmt.Errorf("Incorrect range start %d cannot be bigger than end %d", start, end)
|
||||
}
|
||||
|
||||
for i := start; i <= end; i++ {
|
||||
if strings.HasPrefix(ellipsesRange[0], "0") && len(ellipsesRange[0]) > 1 || strings.HasPrefix(ellipsesRange[1], "0") {
|
||||
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dd", len(ellipsesRange[1])), i))
|
||||
} else {
|
||||
seq = append(seq, fmt.Sprintf("%d", i))
|
||||
}
|
||||
}
|
||||
|
||||
return seq, nil
|
||||
}
|
||||
|
||||
// Pattern - ellipses pattern, describes the range and also the
|
||||
// associated prefix and suffixes.
|
||||
type Pattern struct {
|
||||
Prefix string
|
||||
Suffix string
|
||||
Seq []string
|
||||
}
|
||||
|
||||
// argExpander - recursively expands labels into its respective forms.
|
||||
func argExpander(labels [][]string) (out [][]string) {
|
||||
if len(labels) == 1 {
|
||||
for _, v := range labels[0] {
|
||||
out = append(out, []string{v})
|
||||
}
|
||||
return out
|
||||
}
|
||||
for _, lbl := range labels[0] {
|
||||
rs := argExpander(labels[1:])
|
||||
for _, rlbls := range rs {
|
||||
r := append(rlbls, []string{lbl}...)
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ArgPattern contains a list of patterns provided in the input.
|
||||
type ArgPattern []Pattern
|
||||
|
||||
// Expand - expands all the ellipses patterns in
|
||||
// the given argument.
|
||||
func (a ArgPattern) Expand() [][]string {
|
||||
labels := make([][]string, len(a))
|
||||
for i := range labels {
|
||||
labels[i] = a[i].Expand()
|
||||
}
|
||||
return argExpander(labels)
|
||||
}
|
||||
|
||||
// Expand - expands a ellipses pattern.
|
||||
func (p Pattern) Expand() []string {
|
||||
var labels []string
|
||||
for i := range p.Seq {
|
||||
switch {
|
||||
case p.Prefix != "" && p.Suffix == "":
|
||||
labels = append(labels, fmt.Sprintf("%s%s", p.Prefix, p.Seq[i]))
|
||||
case p.Suffix != "" && p.Prefix == "":
|
||||
labels = append(labels, fmt.Sprintf("%s%s", p.Seq[i], p.Suffix))
|
||||
case p.Suffix == "" && p.Prefix == "":
|
||||
labels = append(labels, fmt.Sprintf("%s", p.Seq[i]))
|
||||
default:
|
||||
labels = append(labels, fmt.Sprintf("%s%s%s", p.Prefix, p.Seq[i], p.Suffix))
|
||||
}
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
// HasEllipses - returns true if input arg has ellipses type pattern.
|
||||
func HasEllipses(args ...string) bool {
|
||||
var ok = true
|
||||
for _, arg := range args {
|
||||
ok = ok && (strings.Count(arg, ellipses) > 0 || (strings.Count(arg, openBraces) > 0 && strings.Count(arg, closeBraces) > 0))
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// ErrInvalidEllipsesFormatFn error returned when invalid ellipses format is detected.
|
||||
var ErrInvalidEllipsesFormatFn = func(arg string) error {
|
||||
return fmt.Errorf("Invalid ellipsis format in (%s), Ellipsis range must be provided in format {N...M} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", arg)
|
||||
}
|
||||
|
||||
// FindEllipsesPatterns - finds all ellipses patterns, recursively and parses the ranges numerically.
|
||||
func FindEllipsesPatterns(arg string) (ArgPattern, error) {
|
||||
var patterns []Pattern
|
||||
parts := regexpEllipses.FindStringSubmatch(arg)
|
||||
if len(parts) == 0 {
|
||||
// We throw an error if arg doesn't have any recognizable ellipses pattern.
|
||||
return nil, ErrInvalidEllipsesFormatFn(arg)
|
||||
}
|
||||
|
||||
parts = parts[1:]
|
||||
patternFound := regexpEllipses.MatchString(parts[0])
|
||||
for patternFound {
|
||||
seq, err := parseEllipsesRange(parts[1])
|
||||
if err != nil {
|
||||
return patterns, err
|
||||
}
|
||||
patterns = append(patterns, Pattern{
|
||||
Prefix: "",
|
||||
Suffix: parts[2],
|
||||
Seq: seq,
|
||||
})
|
||||
parts = regexpEllipses.FindStringSubmatch(parts[0])
|
||||
if len(parts) > 0 {
|
||||
parts = parts[1:]
|
||||
patternFound = HasEllipses(parts[0])
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if len(parts) > 0 {
|
||||
seq, err := parseEllipsesRange(parts[1])
|
||||
if err != nil {
|
||||
return patterns, err
|
||||
}
|
||||
|
||||
patterns = append(patterns, Pattern{
|
||||
Prefix: parts[0],
|
||||
Suffix: parts[2],
|
||||
Seq: seq,
|
||||
})
|
||||
}
|
||||
|
||||
// Check if any of the prefix or suffixes now have flower braces
|
||||
// left over, in such a case we generally think that there is
|
||||
// perhaps a typo in users input and error out accordingly.
|
||||
for _, pattern := range patterns {
|
||||
if strings.Count(pattern.Prefix, openBraces) > 0 || strings.Count(pattern.Prefix, closeBraces) > 0 {
|
||||
return nil, ErrInvalidEllipsesFormatFn(arg)
|
||||
}
|
||||
if strings.Count(pattern.Suffix, openBraces) > 0 || strings.Count(pattern.Suffix, closeBraces) > 0 {
|
||||
return nil, ErrInvalidEllipsesFormatFn(arg)
|
||||
}
|
||||
}
|
||||
|
||||
return patterns, nil
|
||||
}
|
||||
244
pkg/ellipses/ellipses_test.go
Normal file
244
pkg/ellipses/ellipses_test.go
Normal file
@@ -0,0 +1,244 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package ellipses
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Test tests args with ellipses.
|
||||
func TestHasEllipses(t *testing.T) {
|
||||
testCases := []struct {
|
||||
args []string
|
||||
expectedOk bool
|
||||
}{
|
||||
// Tests for all args without ellipses.
|
||||
{
|
||||
[]string{"64"},
|
||||
false,
|
||||
},
|
||||
// Found flower braces, still attempt to parse and throw an error.
|
||||
{
|
||||
[]string{"{1..64}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"{1..2..}"},
|
||||
true,
|
||||
},
|
||||
// Test for valid input.
|
||||
{
|
||||
[]string{"1...64"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"{1...2O}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"..."},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"{-1...1}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"{0...-1}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"{1....4}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"{1...64}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"{...}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"{1...64}", "{65...128}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{"http://minio{2...3}/export/set{1...64}"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{
|
||||
"http://minio{2...3}/export/set{1...64}",
|
||||
"http://minio{2...3}/export/set{65...128}",
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{
|
||||
"mydisk-{a...z}{1...20}",
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
[]string{
|
||||
"mydisk-{1...4}{1..2.}",
|
||||
},
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
|
||||
gotOk := HasEllipses(testCase.args...)
|
||||
if gotOk != testCase.expectedOk {
|
||||
t.Errorf("Expected %t, got %t", testCase.expectedOk, gotOk)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test tests find ellipses patterns.
|
||||
func TestFindEllipsesPatterns(t *testing.T) {
|
||||
testCases := []struct {
|
||||
pattern string
|
||||
success bool
|
||||
expectedCount int
|
||||
}{
|
||||
// Tests for all invalid inputs
|
||||
{
|
||||
"{1..64}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"1...64",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"...",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{1...",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"...64}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{...}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{-1...1}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{0...-1}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{1...2O}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{64...1}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{1....4}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"mydisk-{a...z}{1...20}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"mydisk-{1...4}{1..2.}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{1..2.}-mydisk-{1...4}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{{1...4}}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
{
|
||||
"{4...02}",
|
||||
false,
|
||||
0,
|
||||
},
|
||||
// Test for valid input.
|
||||
{
|
||||
"{1...64}",
|
||||
true,
|
||||
64,
|
||||
},
|
||||
{
|
||||
"{1...64} {65...128}",
|
||||
true,
|
||||
4096,
|
||||
},
|
||||
{
|
||||
"{01...036}",
|
||||
true,
|
||||
36,
|
||||
},
|
||||
{
|
||||
"{001...036}",
|
||||
true,
|
||||
36,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
|
||||
argP, err := FindEllipsesPatterns(testCase.pattern)
|
||||
if err != nil && testCase.success {
|
||||
t.Errorf("Expected success but failed instead %s", err)
|
||||
}
|
||||
if err == nil && !testCase.success {
|
||||
t.Errorf("Expected failure but passed instead")
|
||||
}
|
||||
if err == nil {
|
||||
gotCount := len(argP.Expand())
|
||||
if gotCount != testCase.expectedCount {
|
||||
t.Errorf("Expected %d, got %d", testCase.expectedCount, gotCount)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user