MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Implement bucket expansion (#8509)

Browse Source
This commit is contained in:
Harshavardhana
2019-11-19 17:42:27 -08:00
committed by kannappanr
parent 3a34d98db8
commit 347b29d059
63 changed files with 2208 additions and 1166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
cmd/endpoint-ellipses.go
View File

@@ -58,31 +58,20 @@ func getDivisibleSize(totalSizes []uint64) (result uint64) {
return result
}
// isValidSetSize - checks whether given count is a valid set size for erasure coding.
var isValidSetSize = func(count uint64) bool {
return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1] && count%2 == 0)
}
// getSetIndexes returns list of indexes which provides the set size
// on each index, this function also determines the final set size
// The final set size has the affinity towards choosing smaller
// indexes (total sets)
func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, err error) {
func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint64) (setIndexes [][]uint64, err error) {
if len(totalSizes) == 0 || len(args) == 0 {
return nil, errInvalidArgument
}
// isValidSetSize - checks whether given count is a valid set size for erasure coding.
isValidSetSize := func(count uint64) bool {
return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1] && count%2 == 0)
}
var customSetDriveCount uint64
if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
customSetDriveCount, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, config.ErrInvalidErasureSetSize(err)
}
if !isValidSetSize(customSetDriveCount) {
return nil, config.ErrInvalidErasureSetSize(nil)
}
}
setIndexes = make([][]uint64, len(totalSizes))
for _, totalSize := range totalSizes {
// Check if totalSize has minimum range upto setSize
@@ -189,7 +178,7 @@ func getTotalSizes(argPatterns []ellipses.ArgPattern) []uint64 {
// Parses all arguments and returns an endpointSet which is a collection
// of endpoints following the ellipses pattern, this is what is used
// by the object layer for initializing itself.
func parseEndpointSet(args ...string) (ep endpointSet, err error) {
func parseEndpointSet(customSetDriveCount uint64, args ...string) (ep endpointSet, err error) {
var argPatterns = make([]ellipses.ArgPattern, len(args))
for i, arg := range args {
patterns, perr := ellipses.FindEllipsesPatterns(arg)
@@ -199,7 +188,7 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
argPatterns[i] = patterns
}
ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns))
ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns), customSetDriveCount)
if err != nil {
return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
}
@@ -215,8 +204,15 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
// For example: {1...64} is divided into 4 sets each of size 16.
// This applies to even distributed setup syntax as well.
func GetAllSets(args ...string) ([][]string, error) {
if len(args) == 0 {
return nil, errInvalidArgument
var customSetDriveCount uint64
if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
customSetDriveCount, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, config.ErrInvalidErasureSetSize(err)
}
if !isValidSetSize(customSetDriveCount) {
return nil, config.ErrInvalidErasureSetSize(nil)
}
}
var setArgs [][]string
@@ -225,7 +221,7 @@ func GetAllSets(args ...string) ([][]string, error) {
// Check if we have more one args.
if len(args) > 1 {
var err error
setIndexes, err = getSetIndexes(args, []uint64{uint64(len(args))})
setIndexes, err = getSetIndexes(args, []uint64{uint64(len(args))}, customSetDriveCount)
if err != nil {
return nil, err
}
@@ -239,7 +235,7 @@ func GetAllSets(args ...string) ([][]string, error) {
}
setArgs = s.Get()
} else {
s, err := parseEndpointSet(args...)
s, err := parseEndpointSet(customSetDriveCount, args...)
if err != nil {
return nil, err
}
@@ -261,18 +257,60 @@ func GetAllSets(args ...string) ([][]string, error) {
// CreateServerEndpoints - validates and creates new endpoints from input args, supports
// both ellipses and without ellipses transparently.
func createServerEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, int, int, error) {
setArgs, err := GetAllSets(args...)
if err != nil {
return serverAddr, nil, -1, 0, 0, err
func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, SetupType, error) {
if len(args) == 0 {
return nil, -1, errInvalidArgument
}
var endpoints EndpointList
var endpointZones EndpointZones
var setupType SetupType
serverAddr, endpoints, setupType, err = CreateEndpoints(serverAddr, setArgs...)
if err != nil {
return serverAddr, nil, -1, 0, 0, err
if !ellipses.HasEllipses(args...) {
setArgs, err := GetAllSets(args...)
if err != nil {
return nil, -1, err
}
endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...)
if err != nil {
return nil, -1, err
}
endpointZones = append(endpointZones, ZoneEndpoints{
SetCount: len(setArgs),
DrivesPerSet: len(setArgs[0]),
Endpoints: endpointList,
})
globalXLSetDriveCount = len(setArgs[0])
setupType = newSetupType
return endpointZones, setupType, nil
}
return serverAddr, endpoints, setupType, len(setArgs), len(setArgs[0]), nil
// Look for duplicate args.
if _, err := GetAllSets(args...); err != nil {
return nil, -1, err
}
for _, arg := range args {
setArgs, err := GetAllSets(arg)
if err != nil {
return nil, -1, err
}
endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...)
if err != nil {
return nil, -1, err
}
if setupType != 0 && setupType != newSetupType {
return nil, -1, fmt.Errorf("Mixed modes of operation %s and %s are not allowed",
setupType, newSetupType)
}
if globalXLSetDriveCount != 0 && globalXLSetDriveCount != len(setArgs[0]) {
return nil, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d",
globalXLSetDriveCount, len(setArgs[0]))
}
endpointZones = append(endpointZones, ZoneEndpoints{
SetCount: len(setArgs),
DrivesPerSet: len(setArgs[0]),
Endpoints: endpointList,
})
globalXLSetDriveCount = len(setArgs[0])
setupType = newSetupType
}
return endpointZones, setupType, nil
}