mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
Add large bucket support for erasure coded backend (#5160)
This PR implements an object layer which combines input erasure sets of XL layers into a unified namespace. This object layer extends the existing erasure coded implementation, it is assumed in this design that providing > 16 disks is a static configuration as well i.e if you started the setup with 32 disks with 4 sets 8 disks per pack then you would need to provide 4 sets always. Some design details and restrictions: - Objects are distributed using consistent ordering to a unique erasure coded layer. - Each pack has its own dsync so locks are synchronized properly at pack (erasure layer). - Each pack still has a maximum of 16 disks requirement, you can start with multiple such sets statically. - Static sets set of disks and cannot be changed, there is no elastic expansion allowed. - Static sets set of disks and cannot be changed, there is no elastic removal allowed. - ListObjects() across sets can be noticeably slower since List happens on all servers, and is merged at this sets layer. Fixes #5465 Fixes #5464 Fixes #5461 Fixes #5460 Fixes #5459 Fixes #5458 Fixes #5460 Fixes #5488 Fixes #5489 Fixes #5497 Fixes #5496
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
dd80256151
commit
fb96779a8a
251
cmd/endpoint-ellipses.go
Normal file
251
cmd/endpoint-ellipses.go
Normal file
@@ -0,0 +1,251 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/pkg/ellipses"
|
||||
)
|
||||
|
||||
// This file implements and supports ellipses pattern for
|
||||
// `minio server` command line arguments.
|
||||
|
||||
// Maximum number of unique args supported on the command line.
|
||||
const (
|
||||
serverCommandLineArgsMax = 32
|
||||
)
|
||||
|
||||
// Endpoint set represents parsed ellipses values, also provides
|
||||
// methods to get the sets of endpoints.
|
||||
type endpointSet struct {
|
||||
argPatterns []ellipses.ArgPattern
|
||||
endpoints []string // Endpoints saved from previous GetEndpoints().
|
||||
setIndexes [][]uint64 // All the sets.
|
||||
}
|
||||
|
||||
// Supported set sizes this is used to find the optimal
|
||||
// single set size.
|
||||
var setSizes = []uint64{4, 6, 8, 10, 12, 14, 16}
|
||||
|
||||
// getDivisibleSize - returns a greatest common divisor of
|
||||
// all the ellipses sizes.
|
||||
func getDivisibleSize(totalSizes []uint64) (result uint64) {
|
||||
gcd := func(x, y uint64) uint64 {
|
||||
for y != 0 {
|
||||
x, y = y, x%y
|
||||
}
|
||||
return x
|
||||
}
|
||||
result = totalSizes[0]
|
||||
for i := 1; i < len(totalSizes); i++ {
|
||||
result = gcd(result, totalSizes[i])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// getSetIndexes returns list of indexes which provides the set size
|
||||
// on each index, this function also determines the final set size
|
||||
// The final set size has the affinity towards choosing smaller
|
||||
// indexes (total sets)
|
||||
func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, err error) {
|
||||
if len(totalSizes) == 0 || len(args) == 0 {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
|
||||
setIndexes = make([][]uint64, len(totalSizes))
|
||||
for i, totalSize := range totalSizes {
|
||||
// Check if totalSize has minimum range upto setSize
|
||||
if totalSize < setSizes[0] {
|
||||
return nil, fmt.Errorf("Invalid inputs (%s). Ellipses range or number of args %d should be atleast divisible by least possible set size %d",
|
||||
args[i], totalSize, setSizes[0])
|
||||
}
|
||||
}
|
||||
|
||||
var setSize uint64
|
||||
|
||||
commonSize := getDivisibleSize(totalSizes)
|
||||
if commonSize > setSizes[len(setSizes)-1] {
|
||||
prevD := commonSize / setSizes[0]
|
||||
for _, i := range setSizes {
|
||||
if commonSize%i == 0 {
|
||||
d := commonSize / i
|
||||
if d <= prevD {
|
||||
prevD = d
|
||||
setSize = i
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
setSize = commonSize
|
||||
}
|
||||
|
||||
// isValidSetSize - checks whether given count is a valid set size for erasure coding.
|
||||
isValidSetSize := func(count uint64) bool {
|
||||
return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1] && count%2 == 0)
|
||||
}
|
||||
|
||||
// Check whether setSize is with the supported range.
|
||||
if !isValidSetSize(setSize) {
|
||||
return nil, fmt.Errorf("Invalid inputs (%s). Ellipses range or number of args %d should be atleast divisible by least possible set size %d",
|
||||
args, setSize, setSizes[0])
|
||||
}
|
||||
|
||||
for i := range totalSizes {
|
||||
for j := uint64(0); j < totalSizes[i]/setSize; j++ {
|
||||
setIndexes[i] = append(setIndexes[i], setSize)
|
||||
}
|
||||
}
|
||||
|
||||
return setIndexes, nil
|
||||
}
|
||||
|
||||
// Returns all the expanded endpoints, each argument is expanded separately.
|
||||
func (s endpointSet) getEndpoints() (endpoints []string) {
|
||||
if len(s.endpoints) != 0 {
|
||||
return s.endpoints
|
||||
}
|
||||
for _, argPattern := range s.argPatterns {
|
||||
for _, lbls := range argPattern.Expand() {
|
||||
endpoints = append(endpoints, strings.Join(lbls, ""))
|
||||
}
|
||||
}
|
||||
s.endpoints = endpoints
|
||||
return endpoints
|
||||
}
|
||||
|
||||
// Get returns the sets representation of the endpoints
|
||||
// this function also intelligently decides on what will
|
||||
// be the right set size etc.
|
||||
func (s endpointSet) Get() (sets [][]string) {
|
||||
var k = uint64(0)
|
||||
endpoints := s.getEndpoints()
|
||||
for i := range s.setIndexes {
|
||||
for j := range s.setIndexes[i] {
|
||||
sets = append(sets, endpoints[k:s.setIndexes[i][j]+k])
|
||||
k = s.setIndexes[i][j] + k
|
||||
}
|
||||
}
|
||||
|
||||
return sets
|
||||
}
|
||||
|
||||
// Return the total size for each argument patterns.
|
||||
func getTotalSizes(argPatterns []ellipses.ArgPattern) []uint64 {
|
||||
var totalSizes []uint64
|
||||
for _, argPattern := range argPatterns {
|
||||
var totalSize uint64 = 1
|
||||
for _, p := range argPattern {
|
||||
totalSize = totalSize * uint64(len(p.Seq))
|
||||
}
|
||||
totalSizes = append(totalSizes, totalSize)
|
||||
}
|
||||
return totalSizes
|
||||
}
|
||||
|
||||
// Parses all arguments and returns an endpointSet which is a collection
|
||||
// of endpoints following the ellipses pattern, this is what is used
|
||||
// by the object layer for initializing itself.
|
||||
func parseEndpointSet(args ...string) (ep endpointSet, err error) {
|
||||
var argPatterns = make([]ellipses.ArgPattern, len(args))
|
||||
for i, arg := range args {
|
||||
patterns, err := ellipses.FindEllipsesPatterns(arg)
|
||||
if err != nil {
|
||||
return endpointSet{}, err
|
||||
}
|
||||
argPatterns[i] = patterns
|
||||
}
|
||||
|
||||
ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns))
|
||||
if err != nil {
|
||||
return endpointSet{}, err
|
||||
}
|
||||
|
||||
ep.argPatterns = argPatterns
|
||||
|
||||
return ep, nil
|
||||
}
|
||||
|
||||
// Parses all ellipses input arguments, expands them into corresponding
|
||||
// list of endpoints chunked evenly in accordance with a specific
|
||||
// set size.
|
||||
// For example: {1...64} is divided into 4 sets each of size 16.
|
||||
// This applies to even distributed setup syntax as well.
|
||||
func getAllSets(args ...string) ([][]string, error) {
|
||||
if len(args) == 0 {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
|
||||
var setArgs [][]string
|
||||
if !ellipses.HasEllipses(args...) {
|
||||
var setIndexes [][]uint64
|
||||
// Check if we have more one args.
|
||||
if len(args) > 1 {
|
||||
var err error
|
||||
setIndexes, err = getSetIndexes(args, []uint64{uint64(len(args))})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// We are in FS setup, proceed forward.
|
||||
setIndexes = [][]uint64{[]uint64{uint64(len(args))}}
|
||||
}
|
||||
s := endpointSet{
|
||||
endpoints: args,
|
||||
setIndexes: setIndexes,
|
||||
}
|
||||
setArgs = s.Get()
|
||||
} else {
|
||||
s, err := parseEndpointSet(args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
setArgs = s.Get()
|
||||
}
|
||||
|
||||
uniqueArgs := set.NewStringSet()
|
||||
for _, sargs := range setArgs {
|
||||
for _, arg := range sargs {
|
||||
if uniqueArgs.Contains(arg) {
|
||||
return nil, fmt.Errorf("Input args (%s) has duplicate ellipses", args)
|
||||
}
|
||||
uniqueArgs.Add(arg)
|
||||
}
|
||||
}
|
||||
|
||||
return setArgs, nil
|
||||
}
|
||||
|
||||
// CreateServerEndpoints - validates and creates new endpoints from input args, supports
|
||||
// both ellipses and without ellipses transparently.
|
||||
func createServerEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, int, int, error) {
|
||||
setArgs, err := getAllSets(args...)
|
||||
if err != nil {
|
||||
return serverAddr, nil, -1, 0, 0, err
|
||||
}
|
||||
|
||||
var endpoints EndpointList
|
||||
var setupType SetupType
|
||||
serverAddr, endpoints, setupType, err = CreateEndpoints(serverAddr, setArgs...)
|
||||
if err != nil {
|
||||
return serverAddr, nil, -1, 0, 0, err
|
||||
}
|
||||
|
||||
return serverAddr, endpoints, setupType, len(setArgs), len(setArgs[0]), nil
|
||||
}
|
||||
Reference in New Issue
Block a user