mirror of
https://github.com/minio/minio.git
synced 2025-11-10 14:09:48 -05:00
Add large bucket support for erasure coded backend (#5160)
This PR implements an object layer which combines input erasure sets of XL layers into a unified namespace. This object layer extends the existing erasure coded implementation, it is assumed in this design that providing > 16 disks is a static configuration as well i.e if you started the setup with 32 disks with 4 sets 8 disks per pack then you would need to provide 4 sets always. Some design details and restrictions: - Objects are distributed using consistent ordering to a unique erasure coded layer. - Each pack has its own dsync so locks are synchronized properly at pack (erasure layer). - Each pack still has a maximum of 16 disks requirement, you can start with multiple such sets statically. - Static sets set of disks and cannot be changed, there is no elastic expansion allowed. - Static sets set of disks and cannot be changed, there is no elastic removal allowed. - ListObjects() across sets can be noticeably slower since List happens on all servers, and is merged at this sets layer. Fixes #5465 Fixes #5464 Fixes #5461 Fixes #5460 Fixes #5459 Fixes #5458 Fixes #5460 Fixes #5488 Fixes #5489 Fixes #5497 Fixes #5496
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
dd80256151
commit
fb96779a8a
207
pkg/ellipses/ellipses.go
Normal file
207
pkg/ellipses/ellipses.go
Normal file
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package ellipses
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
// Regex to extract ellipses syntax inputs.
|
||||
regexpEllipses = regexp.MustCompile(`(.*)({[0-9]*\.\.\.[0-9]*})(.*)`)
|
||||
|
||||
// Ellipses constants
|
||||
openBraces = "{"
|
||||
closeBraces = "}"
|
||||
ellipses = "..."
|
||||
)
|
||||
|
||||
// Parses an ellipses range pattern of following style
|
||||
// `{1...64}`
|
||||
// `{33...64}`
|
||||
func parseEllipsesRange(pattern string) (seq []string, err error) {
|
||||
if strings.Index(pattern, openBraces) == -1 {
|
||||
return nil, errors.New("Invalid argument")
|
||||
}
|
||||
if strings.Index(pattern, closeBraces) == -1 {
|
||||
return nil, errors.New("Invalid argument")
|
||||
}
|
||||
|
||||
pattern = strings.TrimPrefix(pattern, openBraces)
|
||||
pattern = strings.TrimSuffix(pattern, closeBraces)
|
||||
|
||||
ellipsesRange := strings.Split(pattern, ellipses)
|
||||
if len(ellipsesRange) != 2 {
|
||||
return nil, errors.New("Invalid argument")
|
||||
}
|
||||
|
||||
var start, end uint64
|
||||
if start, err = strconv.ParseUint(ellipsesRange[0], 10, 64); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if end, err = strconv.ParseUint(ellipsesRange[1], 10, 64); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if start > end {
|
||||
return nil, fmt.Errorf("Incorrect range start %d cannot be bigger than end %d", start, end)
|
||||
}
|
||||
|
||||
for i := start; i <= end; i++ {
|
||||
if strings.HasPrefix(ellipsesRange[0], "0") && len(ellipsesRange[0]) > 1 || strings.HasPrefix(ellipsesRange[1], "0") {
|
||||
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dd", len(ellipsesRange[1])), i))
|
||||
} else {
|
||||
seq = append(seq, fmt.Sprintf("%d", i))
|
||||
}
|
||||
}
|
||||
|
||||
return seq, nil
|
||||
}
|
||||
|
||||
// Pattern - ellipses pattern, describes the range and also the
|
||||
// associated prefix and suffixes.
|
||||
type Pattern struct {
|
||||
Prefix string
|
||||
Suffix string
|
||||
Seq []string
|
||||
}
|
||||
|
||||
// argExpander - recursively expands labels into its respective forms.
|
||||
func argExpander(labels [][]string) (out [][]string) {
|
||||
if len(labels) == 1 {
|
||||
for _, v := range labels[0] {
|
||||
out = append(out, []string{v})
|
||||
}
|
||||
return out
|
||||
}
|
||||
for _, lbl := range labels[0] {
|
||||
rs := argExpander(labels[1:])
|
||||
for _, rlbls := range rs {
|
||||
r := append(rlbls, []string{lbl}...)
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ArgPattern contains a list of patterns provided in the input.
|
||||
type ArgPattern []Pattern
|
||||
|
||||
// Expand - expands all the ellipses patterns in
|
||||
// the given argument.
|
||||
func (a ArgPattern) Expand() [][]string {
|
||||
labels := make([][]string, len(a))
|
||||
for i := range labels {
|
||||
labels[i] = a[i].Expand()
|
||||
}
|
||||
return argExpander(labels)
|
||||
}
|
||||
|
||||
// Expand - expands a ellipses pattern.
|
||||
func (p Pattern) Expand() []string {
|
||||
var labels []string
|
||||
for i := range p.Seq {
|
||||
switch {
|
||||
case p.Prefix != "" && p.Suffix == "":
|
||||
labels = append(labels, fmt.Sprintf("%s%s", p.Prefix, p.Seq[i]))
|
||||
case p.Suffix != "" && p.Prefix == "":
|
||||
labels = append(labels, fmt.Sprintf("%s%s", p.Seq[i], p.Suffix))
|
||||
case p.Suffix == "" && p.Prefix == "":
|
||||
labels = append(labels, fmt.Sprintf("%s", p.Seq[i]))
|
||||
default:
|
||||
labels = append(labels, fmt.Sprintf("%s%s%s", p.Prefix, p.Seq[i], p.Suffix))
|
||||
}
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
// HasEllipses - returns true if input arg has ellipses type pattern.
|
||||
func HasEllipses(args ...string) bool {
|
||||
var ok = true
|
||||
for _, arg := range args {
|
||||
ok = ok && (strings.Count(arg, ellipses) > 0 || (strings.Count(arg, openBraces) > 0 && strings.Count(arg, closeBraces) > 0))
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// ErrInvalidEllipsesFormatFn error returned when invalid ellipses format is detected.
|
||||
var ErrInvalidEllipsesFormatFn = func(arg string) error {
|
||||
return fmt.Errorf("Invalid ellipsis format in (%s), Ellipsis range must be provided in format {N...M} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", arg)
|
||||
}
|
||||
|
||||
// FindEllipsesPatterns - finds all ellipses patterns, recursively and parses the ranges numerically.
|
||||
func FindEllipsesPatterns(arg string) (ArgPattern, error) {
|
||||
var patterns []Pattern
|
||||
parts := regexpEllipses.FindStringSubmatch(arg)
|
||||
if len(parts) == 0 {
|
||||
// We throw an error if arg doesn't have any recognizable ellipses pattern.
|
||||
return nil, ErrInvalidEllipsesFormatFn(arg)
|
||||
}
|
||||
|
||||
parts = parts[1:]
|
||||
patternFound := regexpEllipses.MatchString(parts[0])
|
||||
for patternFound {
|
||||
seq, err := parseEllipsesRange(parts[1])
|
||||
if err != nil {
|
||||
return patterns, err
|
||||
}
|
||||
patterns = append(patterns, Pattern{
|
||||
Prefix: "",
|
||||
Suffix: parts[2],
|
||||
Seq: seq,
|
||||
})
|
||||
parts = regexpEllipses.FindStringSubmatch(parts[0])
|
||||
if len(parts) > 0 {
|
||||
parts = parts[1:]
|
||||
patternFound = HasEllipses(parts[0])
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if len(parts) > 0 {
|
||||
seq, err := parseEllipsesRange(parts[1])
|
||||
if err != nil {
|
||||
return patterns, err
|
||||
}
|
||||
|
||||
patterns = append(patterns, Pattern{
|
||||
Prefix: parts[0],
|
||||
Suffix: parts[2],
|
||||
Seq: seq,
|
||||
})
|
||||
}
|
||||
|
||||
// Check if any of the prefix or suffixes now have flower braces
|
||||
// left over, in such a case we generally think that there is
|
||||
// perhaps a typo in users input and error out accordingly.
|
||||
for _, pattern := range patterns {
|
||||
if strings.Count(pattern.Prefix, openBraces) > 0 || strings.Count(pattern.Prefix, closeBraces) > 0 {
|
||||
return nil, ErrInvalidEllipsesFormatFn(arg)
|
||||
}
|
||||
if strings.Count(pattern.Suffix, openBraces) > 0 || strings.Count(pattern.Suffix, closeBraces) > 0 {
|
||||
return nil, ErrInvalidEllipsesFormatFn(arg)
|
||||
}
|
||||
}
|
||||
|
||||
return patterns, nil
|
||||
}
|
||||
Reference in New Issue
Block a user