mirror of
https://github.com/minio/minio.git
synced 2024-12-25 06:35:56 -05:00
fb96779a8a
This PR implements an object layer which combines input erasure sets of XL layers into a unified namespace. This object layer extends the existing erasure coded implementation, it is assumed in this design that providing > 16 disks is a static configuration as well i.e if you started the setup with 32 disks with 4 sets 8 disks per pack then you would need to provide 4 sets always. Some design details and restrictions: - Objects are distributed using consistent ordering to a unique erasure coded layer. - Each pack has its own dsync so locks are synchronized properly at pack (erasure layer). - Each pack still has a maximum of 16 disks requirement, you can start with multiple such sets statically. - Static sets set of disks and cannot be changed, there is no elastic expansion allowed. - Static sets set of disks and cannot be changed, there is no elastic removal allowed. - ListObjects() across sets can be noticeably slower since List happens on all servers, and is merged at this sets layer. Fixes #5465 Fixes #5464 Fixes #5461 Fixes #5460 Fixes #5459 Fixes #5458 Fixes #5460 Fixes #5488 Fixes #5489 Fixes #5497 Fixes #5496
208 lines
5.7 KiB
Go
208 lines
5.7 KiB
Go
/*
|
|
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package ellipses
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
var (
|
|
// Regex to extract ellipses syntax inputs.
|
|
regexpEllipses = regexp.MustCompile(`(.*)({[0-9]*\.\.\.[0-9]*})(.*)`)
|
|
|
|
// Ellipses constants
|
|
openBraces = "{"
|
|
closeBraces = "}"
|
|
ellipses = "..."
|
|
)
|
|
|
|
// Parses an ellipses range pattern of following style
|
|
// `{1...64}`
|
|
// `{33...64}`
|
|
func parseEllipsesRange(pattern string) (seq []string, err error) {
|
|
if strings.Index(pattern, openBraces) == -1 {
|
|
return nil, errors.New("Invalid argument")
|
|
}
|
|
if strings.Index(pattern, closeBraces) == -1 {
|
|
return nil, errors.New("Invalid argument")
|
|
}
|
|
|
|
pattern = strings.TrimPrefix(pattern, openBraces)
|
|
pattern = strings.TrimSuffix(pattern, closeBraces)
|
|
|
|
ellipsesRange := strings.Split(pattern, ellipses)
|
|
if len(ellipsesRange) != 2 {
|
|
return nil, errors.New("Invalid argument")
|
|
}
|
|
|
|
var start, end uint64
|
|
if start, err = strconv.ParseUint(ellipsesRange[0], 10, 64); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if end, err = strconv.ParseUint(ellipsesRange[1], 10, 64); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if start > end {
|
|
return nil, fmt.Errorf("Incorrect range start %d cannot be bigger than end %d", start, end)
|
|
}
|
|
|
|
for i := start; i <= end; i++ {
|
|
if strings.HasPrefix(ellipsesRange[0], "0") && len(ellipsesRange[0]) > 1 || strings.HasPrefix(ellipsesRange[1], "0") {
|
|
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dd", len(ellipsesRange[1])), i))
|
|
} else {
|
|
seq = append(seq, fmt.Sprintf("%d", i))
|
|
}
|
|
}
|
|
|
|
return seq, nil
|
|
}
|
|
|
|
// Pattern - ellipses pattern, describes the range and also the
|
|
// associated prefix and suffixes.
|
|
type Pattern struct {
|
|
Prefix string
|
|
Suffix string
|
|
Seq []string
|
|
}
|
|
|
|
// argExpander - recursively expands labels into its respective forms.
|
|
func argExpander(labels [][]string) (out [][]string) {
|
|
if len(labels) == 1 {
|
|
for _, v := range labels[0] {
|
|
out = append(out, []string{v})
|
|
}
|
|
return out
|
|
}
|
|
for _, lbl := range labels[0] {
|
|
rs := argExpander(labels[1:])
|
|
for _, rlbls := range rs {
|
|
r := append(rlbls, []string{lbl}...)
|
|
out = append(out, r)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ArgPattern contains a list of patterns provided in the input.
|
|
type ArgPattern []Pattern
|
|
|
|
// Expand - expands all the ellipses patterns in
|
|
// the given argument.
|
|
func (a ArgPattern) Expand() [][]string {
|
|
labels := make([][]string, len(a))
|
|
for i := range labels {
|
|
labels[i] = a[i].Expand()
|
|
}
|
|
return argExpander(labels)
|
|
}
|
|
|
|
// Expand - expands a ellipses pattern.
|
|
func (p Pattern) Expand() []string {
|
|
var labels []string
|
|
for i := range p.Seq {
|
|
switch {
|
|
case p.Prefix != "" && p.Suffix == "":
|
|
labels = append(labels, fmt.Sprintf("%s%s", p.Prefix, p.Seq[i]))
|
|
case p.Suffix != "" && p.Prefix == "":
|
|
labels = append(labels, fmt.Sprintf("%s%s", p.Seq[i], p.Suffix))
|
|
case p.Suffix == "" && p.Prefix == "":
|
|
labels = append(labels, fmt.Sprintf("%s", p.Seq[i]))
|
|
default:
|
|
labels = append(labels, fmt.Sprintf("%s%s%s", p.Prefix, p.Seq[i], p.Suffix))
|
|
}
|
|
}
|
|
return labels
|
|
}
|
|
|
|
// HasEllipses - returns true if input arg has ellipses type pattern.
|
|
func HasEllipses(args ...string) bool {
|
|
var ok = true
|
|
for _, arg := range args {
|
|
ok = ok && (strings.Count(arg, ellipses) > 0 || (strings.Count(arg, openBraces) > 0 && strings.Count(arg, closeBraces) > 0))
|
|
}
|
|
return ok
|
|
}
|
|
|
|
// ErrInvalidEllipsesFormatFn error returned when invalid ellipses format is detected.
|
|
var ErrInvalidEllipsesFormatFn = func(arg string) error {
|
|
return fmt.Errorf("Invalid ellipsis format in (%s), Ellipsis range must be provided in format {N...M} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", arg)
|
|
}
|
|
|
|
// FindEllipsesPatterns - finds all ellipses patterns, recursively and parses the ranges numerically.
|
|
func FindEllipsesPatterns(arg string) (ArgPattern, error) {
|
|
var patterns []Pattern
|
|
parts := regexpEllipses.FindStringSubmatch(arg)
|
|
if len(parts) == 0 {
|
|
// We throw an error if arg doesn't have any recognizable ellipses pattern.
|
|
return nil, ErrInvalidEllipsesFormatFn(arg)
|
|
}
|
|
|
|
parts = parts[1:]
|
|
patternFound := regexpEllipses.MatchString(parts[0])
|
|
for patternFound {
|
|
seq, err := parseEllipsesRange(parts[1])
|
|
if err != nil {
|
|
return patterns, err
|
|
}
|
|
patterns = append(patterns, Pattern{
|
|
Prefix: "",
|
|
Suffix: parts[2],
|
|
Seq: seq,
|
|
})
|
|
parts = regexpEllipses.FindStringSubmatch(parts[0])
|
|
if len(parts) > 0 {
|
|
parts = parts[1:]
|
|
patternFound = HasEllipses(parts[0])
|
|
continue
|
|
}
|
|
break
|
|
}
|
|
|
|
if len(parts) > 0 {
|
|
seq, err := parseEllipsesRange(parts[1])
|
|
if err != nil {
|
|
return patterns, err
|
|
}
|
|
|
|
patterns = append(patterns, Pattern{
|
|
Prefix: parts[0],
|
|
Suffix: parts[2],
|
|
Seq: seq,
|
|
})
|
|
}
|
|
|
|
// Check if any of the prefix or suffixes now have flower braces
|
|
// left over, in such a case we generally think that there is
|
|
// perhaps a typo in users input and error out accordingly.
|
|
for _, pattern := range patterns {
|
|
if strings.Count(pattern.Prefix, openBraces) > 0 || strings.Count(pattern.Prefix, closeBraces) > 0 {
|
|
return nil, ErrInvalidEllipsesFormatFn(arg)
|
|
}
|
|
if strings.Count(pattern.Suffix, openBraces) > 0 || strings.Count(pattern.Suffix, closeBraces) > 0 {
|
|
return nil, ErrInvalidEllipsesFormatFn(arg)
|
|
}
|
|
}
|
|
|
|
return patterns, nil
|
|
}
|