mirror of
https://github.com/minio/minio.git
synced 2025-11-09 21:49:46 -05:00
Add large bucket support for erasure coded backend (#5160)
This PR implements an object layer which combines input erasure sets of XL layers into a unified namespace. This object layer extends the existing erasure coded implementation, it is assumed in this design that providing > 16 disks is a static configuration as well i.e if you started the setup with 32 disks with 4 sets 8 disks per pack then you would need to provide 4 sets always. Some design details and restrictions: - Objects are distributed using consistent ordering to a unique erasure coded layer. - Each pack has its own dsync so locks are synchronized properly at pack (erasure layer). - Each pack still has a maximum of 16 disks requirement, you can start with multiple such sets statically. - Static sets set of disks and cannot be changed, there is no elastic expansion allowed. - Static sets set of disks and cannot be changed, there is no elastic removal allowed. - ListObjects() across sets can be noticeably slower since List happens on all servers, and is merged at this sets layer. Fixes #5465 Fixes #5464 Fixes #5461 Fixes #5460 Fixes #5459 Fixes #5458 Fixes #5460 Fixes #5488 Fixes #5489 Fixes #5497 Fixes #5496
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
dd80256151
commit
fb96779a8a
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -16,175 +16,12 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
func listDirHealFactory(isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc {
|
||||
// Returns sorted merged entries from all the disks.
|
||||
listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
|
||||
for _, disk := range disks {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
var entries []string
|
||||
var newEntries []string
|
||||
entries, err = disk.ListDir(bucket, prefixDir)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// isLeaf() check has to happen here so that
|
||||
// trailing "/" for objects can be removed.
|
||||
for i, entry := range entries {
|
||||
if isLeaf(bucket, pathJoin(prefixDir, entry)) {
|
||||
entries[i] = strings.TrimSuffix(entry, slashSeparator)
|
||||
}
|
||||
}
|
||||
|
||||
// Find elements in entries which are not in mergedEntries
|
||||
for _, entry := range entries {
|
||||
idx := sort.SearchStrings(mergedEntries, entry)
|
||||
// if entry is already present in mergedEntries don't add.
|
||||
if idx < len(mergedEntries) && mergedEntries[idx] == entry {
|
||||
continue
|
||||
}
|
||||
newEntries = append(newEntries, entry)
|
||||
}
|
||||
|
||||
if len(newEntries) > 0 {
|
||||
// Merge the entries and sort it.
|
||||
mergedEntries = append(mergedEntries, newEntries...)
|
||||
sort.Strings(mergedEntries)
|
||||
}
|
||||
|
||||
// Filter entries that have the prefix prefixEntry.
|
||||
mergedEntries = filterMatchingPrefix(mergedEntries, prefixEntry)
|
||||
}
|
||||
return mergedEntries, false, nil
|
||||
}
|
||||
return listDir
|
||||
// This is not implemented/needed anymore, look for xl-sets.ListBucketHeal()
|
||||
func (xl xlObjects) ListBucketsHeal() ([]BucketInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// listObjectsHeal - wrapper function implemented over file tree walk.
|
||||
func (xl xlObjects) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
// Default is recursive, if delimiter is set then list non recursive.
|
||||
recursive := true
|
||||
if delimiter == slashSeparator {
|
||||
recursive = false
|
||||
}
|
||||
|
||||
// "heal" true for listObjectsHeal() and false for listObjects()
|
||||
heal := true
|
||||
walkResultCh, endWalkCh := xl.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
|
||||
if walkResultCh == nil {
|
||||
endWalkCh = make(chan struct{})
|
||||
isLeaf := xl.isObject
|
||||
listDir := listDirHealFactory(isLeaf, xl.storageDisks...)
|
||||
walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, nil, endWalkCh)
|
||||
}
|
||||
|
||||
var objInfos []ObjectInfo
|
||||
var eof bool
|
||||
var nextMarker string
|
||||
for i := 0; i < maxKeys; {
|
||||
walkResult, ok := <-walkResultCh
|
||||
if !ok {
|
||||
// Closed channel.
|
||||
eof = true
|
||||
break
|
||||
}
|
||||
// For any walk error return right away.
|
||||
if walkResult.err != nil {
|
||||
return loi, toObjectErr(walkResult.err, bucket, prefix)
|
||||
}
|
||||
entry := walkResult.entry
|
||||
var objInfo ObjectInfo
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
// Object name needs to be full path.
|
||||
objInfo.Bucket = bucket
|
||||
objInfo.Name = entry
|
||||
objInfo.IsDir = true
|
||||
} else {
|
||||
var err error
|
||||
objInfo, err = xl.getObjectInfo(bucket, entry)
|
||||
if err != nil {
|
||||
// Ignore errFileNotFound
|
||||
if errors.Cause(err) == errFileNotFound {
|
||||
continue
|
||||
}
|
||||
return loi, toObjectErr(err, bucket, prefix)
|
||||
}
|
||||
}
|
||||
nextMarker = objInfo.Name
|
||||
objInfos = append(objInfos, objInfo)
|
||||
i++
|
||||
if walkResult.end {
|
||||
eof = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
params := listParams{bucket, recursive, nextMarker, prefix, heal}
|
||||
if !eof {
|
||||
xl.listPool.Set(params, walkResultCh, endWalkCh)
|
||||
}
|
||||
|
||||
result := ListObjectsInfo{IsTruncated: !eof}
|
||||
for _, objInfo := range objInfos {
|
||||
result.NextMarker = objInfo.Name
|
||||
if objInfo.IsDir {
|
||||
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
// Add each object seen to the result - objects are
|
||||
// checked for healing later.
|
||||
result.Objects = append(result.Objects, ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: objInfo.Name,
|
||||
ModTime: objInfo.ModTime,
|
||||
Size: objInfo.Size,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListObjects - list all objects at prefix, delimited by '/'.
|
||||
// This is not implemented/needed anymore, look for xl-sets.ListObjectsHeal()
|
||||
func (xl xlObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
if err := checkListObjsArgs(bucket, prefix, marker, delimiter, xl); err != nil {
|
||||
return loi, err
|
||||
}
|
||||
|
||||
// With max keys of zero we have reached eof, return right here.
|
||||
if maxKeys == 0 {
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
// For delimiter and prefix as '/' we do not list anything at all
|
||||
// since according to s3 spec we stop at the 'delimiter' along
|
||||
// with the prefix. On a flat namespace with 'prefix' as '/'
|
||||
// we don't have any entries, since all the keys are of form 'keyName/...'
|
||||
if delimiter == slashSeparator && prefix == slashSeparator {
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
// Over flowing count - reset to maxObjectList.
|
||||
if maxKeys < 0 || maxKeys > maxObjectList {
|
||||
maxKeys = maxObjectList
|
||||
}
|
||||
|
||||
// Initiate a list operation, if successful filter and return quickly.
|
||||
listObjInfo, err := xl.listObjectsHeal(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err == nil {
|
||||
// We got the entries successfully return.
|
||||
return listObjInfo, nil
|
||||
}
|
||||
|
||||
// Return error at the end.
|
||||
return loi, toObjectErr(err, bucket, prefix)
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user