MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

heal: Dangling check to evaluate object parts separately (#19797)

This commit is contained in:
Anis Eleuch 2024-06-10 16:51:27 +01:00 committed by GitHub
parent 0662c90b5c
commit 789cbc6fb2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 614 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
cmd/erasure-healing-common.go
View File

@ -20,6 +20,7 @@ package cmd
import (
"bytes"
"context"
"slices"
"time"
"github.com/minio/madmin-go/v3"
@ -253,6 +254,33 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error,
return onlineDisks, modTime, ""
}
// Convert verify or check parts returned error to integer representation
func convPartErrToInt(err error) int {
err = unwrapAll(err)
switch err {
case nil:
return checkPartSuccess
case errFileNotFound, errFileVersionNotFound:
return checkPartFileNotFound
case errFileCorrupt:
return checkPartFileCorrupt
case errVolumeNotFound:
return checkPartVolumeNotFound
case errDiskNotFound:
return checkPartDiskNotFound
default:
return checkPartUnknown
}
}
func partNeedsHealing(partErrs []int) bool {
return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess && i != checkPartUnknown }) > -1
}
func hasPartErr(partErrs []int) bool {
return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess }) > -1
}
// disksWithAllParts - This function needs to be called with
// []StorageAPI returned by listOnlineDisks. Returns,
//
@ -262,10 +290,19 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error,
// a not-found error or a hash-mismatch error.
func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
errs []error, latestMeta FileInfo, bucket, object string,
scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time,
) {
availableDisks := make([]StorageAPI, len(onlineDisks))
dataErrs := make([]error, len(onlineDisks))
scanMode madmin.HealScanMode,
) (availableDisks []StorageAPI, dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) {
availableDisks = make([]StorageAPI, len(onlineDisks))
dataErrsByDisk = make(map[int][]int, len(onlineDisks))
for i := range onlineDisks {
dataErrsByDisk[i] = make([]int, len(latestMeta.Parts))
}
dataErrsByPart = make(map[int][]int, len(latestMeta.Parts))
for i := range latestMeta.Parts {
dataErrsByPart[i] = make([]int, len(onlineDisks))
}
inconsistent := 0
for i, meta := range partsMetadata {
@ -295,19 +332,21 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
erasureDistributionReliable = false
}
metaErrs := make([]error, len(errs))
for i, onlineDisk := range onlineDisks {
if errs[i] != nil {
dataErrs[i] = errs[i]
metaErrs[i] = errs[i]
continue
}
if onlineDisk == OfflineDisk {
dataErrs[i] = errDiskNotFound
metaErrs[i] = errDiskNotFound
continue
}
meta := partsMetadata[i]
if !meta.ModTime.Equal(latestMeta.ModTime) || meta.DataDir != latestMeta.DataDir {
dataErrs[i] = errFileCorrupt
metaErrs[i] = errFileCorrupt
partsMetadata[i] = FileInfo{}
continue
}
@ -315,7 +354,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
if erasureDistributionReliable {
if !meta.IsValid() {
partsMetadata[i] = FileInfo{}
dataErrs[i] = errFileCorrupt
metaErrs[i] = errFileCorrupt
continue
}
@ -325,46 +364,79 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
// attempt a fix if possible, assuming other entries
// might have the right erasure distribution.
partsMetadata[i] = FileInfo{}
dataErrs[i] = errFileCorrupt
metaErrs[i] = errFileCorrupt
continue
}
}
}
}
// Copy meta errors to part errors
for i, err := range metaErrs {
if err != nil {
partErr := convPartErrToInt(err)
for p := range latestMeta.Parts {
dataErrsByPart[p][i] = partErr
}
}
}
for i, onlineDisk := range onlineDisks {
if metaErrs[i] != nil {
continue
}
meta := partsMetadata[i]
if meta.Deleted || meta.IsRemote() {
continue
}
// Always check data, if we got it.
if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 {
checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number)
dataErrs[i] = bitrotVerify(bytes.NewReader(meta.Data),
verifyErr := bitrotVerify(bytes.NewReader(meta.Data),
int64(len(meta.Data)),
meta.Erasure.ShardFileSize(meta.Size),
checksumInfo.Algorithm,
checksumInfo.Hash, meta.Erasure.ShardSize())
if dataErrs[i] == nil {
// All parts verified, mark it as all data available.
availableDisks[i] = onlineDisk
} else {
// upon errors just make that disk's fileinfo invalid
partsMetadata[i] = FileInfo{}
}
dataErrsByPart[0][i] = convPartErrToInt(verifyErr)
continue
}
var (
verifyErr error
verifyResp *CheckPartsResp
)
meta.DataDir = latestMeta.DataDir
switch scanMode {
case madmin.HealDeepScan:
// disk has a valid xl.meta but may not have all the
// parts. This is considered an outdated disk, since
// it needs healing too.
if !meta.Deleted && !meta.IsRemote() {
dataErrs[i] = onlineDisk.VerifyFile(ctx, bucket, object, meta)
}
case madmin.HealNormalScan:
if !meta.Deleted && !meta.IsRemote() {
dataErrs[i] = onlineDisk.CheckParts(ctx, bucket, object, meta)
}
verifyResp, verifyErr = onlineDisk.VerifyFile(ctx, bucket, object, meta)
default:
verifyResp, verifyErr = onlineDisk.CheckParts(ctx, bucket, object, meta)
}
if dataErrs[i] == nil {
for p := range latestMeta.Parts {
if verifyErr != nil {
dataErrsByPart[p][i] = convPartErrToInt(verifyErr)
} else {
dataErrsByPart[p][i] = verifyResp.Results[p]
}
}
}
// Build dataErrs by disk from dataErrs by part
for part, disks := range dataErrsByPart {
for disk := range disks {
dataErrsByDisk[disk][part] = dataErrsByPart[part][disk]
}
}
for i, onlineDisk := range onlineDisks {
if metaErrs[i] == nil && !hasPartErr(dataErrsByDisk[i]) {
// All parts verified, mark it as all data available.
availableDisks[i] = onlineDisk
} else {
@ -373,5 +445,5 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
}
}
return availableDisks, dataErrs, timeSentinel
return
}

22
cmd/erasure-healing-common_test.go
View File

@ -308,9 +308,8 @@ func TestListOnlineDisks(t *testing.T) {
t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
test.expectedTime, modTime)
}
availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
test.errs, fi, bucket, object, madmin.HealDeepScan)
test.errs = newErrs
if test._tamperBackend != noTamper {
if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
@ -491,9 +490,8 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
test.expectedTime, modTime)
}
availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
test.errs, fi, bucket, object, madmin.HealDeepScan)
test.errs = newErrs
if test._tamperBackend != noTamper {
if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
@ -554,7 +552,7 @@ func TestDisksWithAllParts(t *testing.T) {
erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum)
filteredDisks, errs, _ := disksWithAllParts(ctx, erasureDisks, partsMetadata,
filteredDisks, _, dataErrsPerDisk := disksWithAllParts(ctx, erasureDisks, partsMetadata,
errs, fi, bucket, object, madmin.HealDeepScan)
if len(filteredDisks) != len(erasureDisks) {
@ -562,8 +560,8 @@ func TestDisksWithAllParts(t *testing.T) {
}
for diskIndex, disk := range filteredDisks {
if errs[diskIndex] != nil {
t.Errorf("Unexpected error %s", errs[diskIndex])
if partNeedsHealing(dataErrsPerDisk[diskIndex]) {
t.Errorf("Unexpected error: %v", dataErrsPerDisk[diskIndex])
}
if disk == nil {
@ -634,7 +632,7 @@ func TestDisksWithAllParts(t *testing.T) {
}
errs = make([]error, len(erasureDisks))
filteredDisks, errs, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
filteredDisks, dataErrsPerDisk, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
errs, fi, bucket, object, madmin.HealDeepScan)
if len(filteredDisks) != len(erasureDisks) {
@ -646,15 +644,15 @@ func TestDisksWithAllParts(t *testing.T) {
if disk != nil {
t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
}
if errs[diskIndex] == nil {
t.Errorf("Expected error not received, driveIndex: %d", diskIndex)
if !partNeedsHealing(dataErrsPerDisk[diskIndex]) {
t.Errorf("Disk expected to be healed, driveIndex: %d", diskIndex)
}
} else {
if disk == nil {
t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
}
if errs[diskIndex] != nil {
t.Errorf("Unexpected error, %s, driveIndex: %d", errs[diskIndex], diskIndex)
if partNeedsHealing(dataErrsPerDisk[diskIndex]) {
t.Errorf("Disk not expected to be healed, driveIndex: %d", diskIndex)
}
}

128
cmd/erasure-healing.go
View File

@ -32,6 +32,7 @@ import (
"github.com/minio/minio/internal/grid"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/sync/errgroup"
"golang.org/x/exp/slices"
)
//go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE
@ -144,36 +145,41 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
return reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum)
}
var errLegacyXLMeta = errors.New("legacy XL meta")
var errOutdatedXLMeta = errors.New("outdated XL meta")
var errPartMissingOrCorrupt = errors.New("part missing or corrupt")
// Only heal on disks where we are sure that healing is needed. We can expand
// this list as and when we figure out more errors can be added to this list safely.
func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo) bool {
switch {
case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
return true
case errors.Is(erErr, errFileCorrupt):
return true
func shouldHealObjectOnDisk(erErr error, partsErrs []int, meta FileInfo, latestMeta FileInfo) (bool, error) {
if errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound) || errors.Is(erErr, errFileCorrupt) {
return true, erErr
}
if erErr == nil {
if meta.XLV1 {
// Legacy means heal always
// always check first.
return true
return true, errLegacyXLMeta
}
if !latestMeta.Equals(meta) {
return true, errOutdatedXLMeta
}
if !meta.Deleted && !meta.IsRemote() {
// If xl.meta was read fine but there may be problem with the part.N files.
if IsErr(dataErr, []error{
errFileNotFound,
errFileVersionNotFound,
errFileCorrupt,
}...) {
return true
for _, partErr := range partsErrs {
if slices.Contains([]int{
checkPartFileNotFound,
checkPartFileCorrupt,
}, partErr) {
return true, errPartMissingOrCorrupt
}
}
}
if !latestMeta.Equals(meta) {
return true
}
return false, nil
}
return false
return false, erErr
}
const (
@ -332,7 +338,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
// used here for reconstruction. This is done to ensure that
// we do not skip drives that have inconsistent metadata to be
// skipped from purging when they are stale.
availableDisks, dataErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
availableDisks, dataErrsByDisk, dataErrsByPart := disksWithAllParts(ctx, onlineDisks, partsMetadata,
errs, latestMeta, bucket, object, scanMode)
var erasure Erasure
@ -355,15 +361,20 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
// to be healed.
outDatedDisks := make([]StorageAPI, len(storageDisks))
disksToHealCount := 0
for i, v := range availableDisks {
for i := range availableDisks {
yes, reason := shouldHealObjectOnDisk(errs[i], dataErrsByDisk[i], partsMetadata[i], latestMeta)
if yes {
outDatedDisks[i] = storageDisks[i]
disksToHealCount++
}
driveState := ""
switch {
case v != nil:
case reason == nil:
driveState = madmin.DriveStateOk
case errors.Is(errs[i], errDiskNotFound), errors.Is(dataErrs[i], errDiskNotFound):
case IsErr(reason, errDiskNotFound):
driveState = madmin.DriveStateOffline
case IsErr(errs[i], errFileNotFound, errFileVersionNotFound, errVolumeNotFound),
IsErr(dataErrs[i], errFileNotFound, errFileVersionNotFound, errVolumeNotFound):
case IsErr(reason, errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errPartMissingOrCorrupt, errOutdatedXLMeta, errLegacyXLMeta):
driveState = madmin.DriveStateMissing
default:
// all remaining cases imply corrupt data/metadata
@ -380,12 +391,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
Endpoint: storageEndpoints[i].String(),
State: driveState,
})
if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
outDatedDisks[i] = storageDisks[i]
disksToHealCount++
continue
}
}
if isAllNotFound(errs) {
@ -412,7 +417,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
// Allow for dangling deletes, on versions that have DataDir missing etc.
// this would end up restoring the correct readable versions.
m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrs, ObjectOptions{
m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrsByPart, ObjectOptions{
VersionID: versionID,
})
errs = make([]error, len(errs))
@ -908,35 +913,52 @@ func isObjectDirDangling(errs []error) (ok bool) {
return found < notFound && found > 0
}
func danglingMetaErrsCount(cerrs []error) (notFoundCount int, nonActionableCount int) {
for _, readErr := range cerrs {
if readErr == nil {
continue
}
switch {
case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
notFoundCount++
default:
// All other errors are non-actionable
nonActionableCount++
}
}
return
}
func danglingPartErrsCount(results []int) (notFoundCount int, nonActionableCount int) {
for _, partResult := range results {
switch partResult {
case checkPartSuccess:
continue
case checkPartFileNotFound:
notFoundCount++
default:
// All other errors are non-actionable
nonActionableCount++
}
}
return
}
// Object is considered dangling/corrupted if and only
// if total disks - a combination of corrupted and missing
// files is lesser than number of data blocks.
func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (validMeta FileInfo, ok bool) {
func isObjectDangling(metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int) (validMeta FileInfo, ok bool) {
// We can consider an object data not reliable
// when xl.meta is not found in read quorum disks.
// or when xl.meta is not readable in read quorum disks.
danglingErrsCount := func(cerrs []error) (int, int) {
var (
notFoundCount int
nonActionableCount int
)
for _, readErr := range cerrs {
if readErr == nil {
continue
}
switch {
case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
notFoundCount++
default:
// All other errors are non-actionable
nonActionableCount++
}
}
return notFoundCount, nonActionableCount
}
notFoundMetaErrs, nonActionableMetaErrs := danglingMetaErrsCount(errs)
notFoundMetaErrs, nonActionableMetaErrs := danglingErrsCount(errs)
notFoundPartsErrs, nonActionablePartsErrs := danglingErrsCount(dataErrs)
notFoundPartsErrs, nonActionablePartsErrs := 0, 0
for _, dataErrs := range dataErrsByPart {
if nf, na := danglingPartErrsCount(dataErrs); nf > notFoundPartsErrs {
notFoundPartsErrs, nonActionablePartsErrs = nf, na
}
}
for _, m := range metaArr {
if m.IsValid() {
@ -948,7 +970,7 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
if !validMeta.IsValid() {
// validMeta is invalid because all xl.meta is missing apparently
// we should figure out if dataDirs are also missing > dataBlocks.
dataBlocks := (len(dataErrs) + 1) / 2
dataBlocks := (len(metaArr) + 1) / 2
if notFoundPartsErrs > dataBlocks {
// Not using parity to ensure that we do not delete
// any valid content, if any is recoverable. But if

69
cmd/erasure-healing_test.go
View File

@ -49,7 +49,7 @@ func TestIsObjectDangling(t *testing.T) {
name string
metaArr []FileInfo
errs []error
dataErrs []error
dataErrs map[int][]int
expectedMeta FileInfo
expectedDangling bool
}{
@ -165,11 +165,8 @@ func TestIsObjectDangling(t *testing.T) {
nil,
nil,
},
dataErrs: []error{
errFileCorrupt,
errFileNotFound,
nil,
errFileCorrupt,
dataErrs: map[int][]int{
0: {checkPartFileCorrupt, checkPartFileNotFound, checkPartSuccess, checkPartFileCorrupt},
},
expectedMeta: fi,
expectedDangling: false,
@ -188,11 +185,8 @@ func TestIsObjectDangling(t *testing.T) {
errFileNotFound,
nil,
},
dataErrs: []error{
errFileNotFound,
errFileCorrupt,
nil,
nil,
dataErrs: map[int][]int{
0: {checkPartFileNotFound, checkPartFileCorrupt, checkPartSuccess, checkPartSuccess},
},
expectedMeta: fi,
expectedDangling: false,
@ -247,15 +241,58 @@ func TestIsObjectDangling(t *testing.T) {
nil,
nil,
},
dataErrs: []error{
errFileNotFound,
errFileNotFound,
nil,
errFileNotFound,
dataErrs: map[int][]int{
0: {checkPartFileNotFound, checkPartFileNotFound, checkPartSuccess, checkPartFileNotFound},
},
expectedMeta: fi,
expectedDangling: true,
},
{
name: "FileInfoDecided-case4-(missing data-dir for part 2)",
metaArr: []FileInfo{
{},
{},
{},
fi,
},
errs: []error{
errFileNotFound,
errFileNotFound,
nil,
nil,
},
dataErrs: map[int][]int{
0: {checkPartSuccess, checkPartSuccess, checkPartSuccess, checkPartSuccess},
1: {checkPartSuccess, checkPartFileNotFound, checkPartFileNotFound, checkPartFileNotFound},
},
expectedMeta: fi,
expectedDangling: true,
},
{
name: "FileInfoDecided-case4-(enough data-dir existing for each part)",
metaArr: []FileInfo{
{},
{},
{},
fi,
},
errs: []error{
errFileNotFound,
errFileNotFound,
nil,
nil,
},
dataErrs: map[int][]int{
0: {checkPartFileNotFound, checkPartSuccess, checkPartSuccess, checkPartSuccess},
1: {checkPartSuccess, checkPartFileNotFound, checkPartSuccess, checkPartSuccess},
2: {checkPartSuccess, checkPartSuccess, checkPartFileNotFound, checkPartSuccess},
3: {checkPartSuccess, checkPartSuccess, checkPartSuccess, checkPartFileNotFound},
},
expectedMeta: fi,
expectedDangling: false,
},
// Add new cases as seen
}
for _, testCase := range testCases {

22
cmd/erasure-object.go
View File

@ -484,8 +484,8 @@ func joinErrs(errs []error) []string {
return s
}
func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) {
m, ok := isObjectDangling(metaArr, errs, dataErrs)
func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int, opts ObjectOptions) (FileInfo, error) {
m, ok := isObjectDangling(metaArr, errs, dataErrsByPart)
if !ok {
// We only come here if we cannot figure out if the object
// can be deleted safely, in such a scenario return ReadQuorum error.
@ -495,7 +495,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
tags["set"] = er.setIndex
tags["pool"] = er.poolIndex
tags["merrs"] = joinErrs(errs)
tags["derrs"] = joinErrs(dataErrs)
tags["derrs"] = dataErrsByPart
if m.IsValid() {
tags["size"] = m.Size
tags["mtime"] = m.ModTime.Format(http.TimeFormat)
@ -509,8 +509,20 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
// count the number of offline disks
offline := 0
for i := 0; i < max(len(errs), len(dataErrs)); i++ {
if i < len(errs) && errors.Is(errs[i], errDiskNotFound) || i < len(dataErrs) && errors.Is(dataErrs[i], errDiskNotFound) {
for i := 0; i < len(errs); i++ {
var found bool
switch {
case errors.Is(errs[i], errDiskNotFound):
found = true
default:
for p := range dataErrsByPart {
if dataErrsByPart[p][i] == checkPartDiskNotFound {
found = true
break
}
}
}
if found {
offline++
}
}

8
cmd/naughty-disk_test.go
View File

@ -215,9 +215,9 @@ func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVol
return d.disk.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
}
func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
if err := d.calcError(); err != nil {
return err
return nil, err
}
return d.disk.CheckParts(ctx, volume, path, fi)
}
@ -289,9 +289,9 @@ func (d *naughtyDisk) ReadXL(ctx context.Context, volume string, path string, re
return d.disk.ReadXL(ctx, volume, path, readData)
}
func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
if err := d.calcError(); err != nil {
return err
return nil, err
}
return d.disk.VerifyFile(ctx, volume, path, fi)
}

17
cmd/storage-datatypes.go
View File

@ -502,6 +502,23 @@ type RenameDataResp struct {
OldDataDir string // contains '<uuid>', it is designed to be passed as value to Delete(bucket, pathJoin(object, dataDir))
}
const (
checkPartUnknown int = iota
// Changing the order can cause a data loss
// when running two nodes with incompatible versions
checkPartSuccess
checkPartDiskNotFound
checkPartVolumeNotFound
checkPartFileNotFound
checkPartFileCorrupt
)
// CheckPartsResp is a response of the storage CheckParts and VerifyFile APIs
type CheckPartsResp struct {
Results []int
}
// LocalDiskIDs - GetLocalIDs response.
type LocalDiskIDs struct {
IDs []string

139
cmd/storage-datatypes_gen.go
View File

@ -273,6 +273,145 @@ func (z *CheckPartsHandlerParams) Msgsize() (s int) {
return
}
// DecodeMsg implements msgp.Decodable
func (z *CheckPartsResp) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "Results":
var zb0002 uint32
zb0002, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "Results")
return
}
if cap(z.Results) >= int(zb0002) {
z.Results = (z.Results)[:zb0002]
} else {
z.Results = make([]int, zb0002)
}
for za0001 := range z.Results {
z.Results[za0001], err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "Results", za0001)
return
}
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *CheckPartsResp) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 1
// write "Results"
err = en.Append(0x81, 0xa7, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73)
if err != nil {
return
}
err = en.WriteArrayHeader(uint32(len(z.Results)))
if err != nil {
err = msgp.WrapError(err, "Results")
return
}
for za0001 := range z.Results {
err = en.WriteInt(z.Results[za0001])
if err != nil {
err = msgp.WrapError(err, "Results", za0001)
return
}
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *CheckPartsResp) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 1
// string "Results"
o = append(o, 0x81, 0xa7, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73)
o = msgp.AppendArrayHeader(o, uint32(len(z.Results)))
for za0001 := range z.Results {
o = msgp.AppendInt(o, z.Results[za0001])
}
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *CheckPartsResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "Results":
var zb0002 uint32
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Results")
return
}
if cap(z.Results) >= int(zb0002) {
z.Results = (z.Results)[:zb0002]
} else {
z.Results = make([]int, zb0002)
}
for za0001 := range z.Results {
z.Results[za0001], bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Results", za0001)
return
}
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *CheckPartsResp) Msgsize() (s int) {
s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.Results) * (msgp.IntSize))
return
}
// DecodeMsg implements msgp.Decodable
func (z *DeleteFileHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte

113
cmd/storage-datatypes_gen_test.go
View File

@ -235,6 +235,119 @@ func BenchmarkDecodeCheckPartsHandlerParams(b *testing.B) {
}
}
func TestMarshalUnmarshalCheckPartsResp(t *testing.T) {
v := CheckPartsResp{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgCheckPartsResp(b *testing.B) {
v := CheckPartsResp{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgCheckPartsResp(b *testing.B) {
v := CheckPartsResp{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalCheckPartsResp(b *testing.B) {
v := CheckPartsResp{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeCheckPartsResp(t *testing.T) {
v := CheckPartsResp{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeCheckPartsResp Msgsize() is inaccurate")
}
vn := CheckPartsResp{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeCheckPartsResp(b *testing.B) {
v := CheckPartsResp{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeCheckPartsResp(b *testing.B) {
v := CheckPartsResp{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
func TestMarshalUnmarshalDeleteFileHandlerParams(t *testing.T) {
v := DeleteFileHandlerParams{}
bts, err := v.MarshalMsg(nil)

4
cmd/storage-interface.go
View File

@ -94,9 +94,9 @@ type StorageAPI interface {
CreateFile(ctx context.Context, origvolume, olume, path string, size int64, reader io.Reader) error
ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error)
RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error
CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error)
Delete(ctx context.Context, volume string, path string, opts DeleteOptions) (err error)
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error)
StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error
CleanAbandonedData(ctx context.Context, volume string, path string) error

24
cmd/storage-rest-client.go
View File

@ -449,14 +449,18 @@ func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, pa
}
// CheckParts - stat all file parts.
func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
_, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{
func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
var resp *CheckPartsResp
resp, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{
DiskID: *client.diskID.Load(),
Volume: volume,
FilePath: path,
FI: fi,
})
return toStorageErr(err)
if err != nil {
return nil, err
}
return resp, nil
}
// RenameData - rename source path to destination path atomically, metadata and data file.
@ -748,33 +752,33 @@ func (client *storageRESTClient) RenameFile(ctx context.Context, srcVolume, srcP
return toStorageErr(err)
}
func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
var reader bytes.Buffer
if err := msgp.Encode(&reader, &fi); err != nil {
return err
return nil, err
}
respBody, err := client.call(ctx, storageRESTMethodVerifyFile, values, &reader, -1)
defer xhttp.DrainBody(respBody)
if err != nil {
return err
return nil, err
}
respReader, err := waitForHTTPResponse(respBody)
if err != nil {
return toStorageErr(err)
return nil, toStorageErr(err)
}
verifyResp := &VerifyFileResp{}
verifyResp := &CheckPartsResp{}
if err = gob.NewDecoder(respReader).Decode(verifyResp); err != nil {
return toStorageErr(err)
return nil, toStorageErr(err)
}
return toStorageErr(verifyResp.Err)
return verifyResp, nil
}
func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {

2
cmd/storage-rest-common.go
View File

@ -20,7 +20,7 @@ package cmd
//go:generate msgp -file $GOFILE -unexported
const (
storageRESTVersion = "v57" // Remove TotalTokens from DiskMetrics
storageRESTVersion = "v58" // Change VerifyFile signature
storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
storageRESTPrefix = minioReservedBucketPath + "/storage"
)

25
cmd/storage-rest-server.go
View File

@ -58,7 +58,7 @@ type storageRESTServer struct {
}
var (
storageCheckPartsRPC = grid.NewSingleHandler[*CheckPartsHandlerParams, grid.NoPayload](grid.HandlerCheckParts, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, grid.NewNoPayload)
storageCheckPartsRPC = grid.NewSingleHandler[*CheckPartsHandlerParams, *CheckPartsResp](grid.HandlerCheckParts2, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, func() *CheckPartsResp { return &CheckPartsResp{} })
storageDeleteFileRPC = grid.NewSingleHandler[*DeleteFileHandlerParams, grid.NoPayload](grid.HandlerDeleteFile, func() *DeleteFileHandlerParams { return &DeleteFileHandlerParams{} }, grid.NewNoPayload).AllowCallRequestPool(true)
storageDeleteVersionRPC = grid.NewSingleHandler[*DeleteVersionHandlerParams, grid.NoPayload](grid.HandlerDeleteVersion, func() *DeleteVersionHandlerParams { return &DeleteVersionHandlerParams{} }, grid.NewNoPayload)
storageDiskInfoRPC = grid.NewSingleHandler[*DiskInfoOptions, *DiskInfo](grid.HandlerDiskInfo, func() *DiskInfoOptions { return &DiskInfoOptions{} }, func() *DiskInfo { return &DiskInfo{} }).WithSharedResponse().AllowCallRequestPool(true)
@ -439,13 +439,15 @@ func (s *storageRESTServer) UpdateMetadataHandler(p *MetadataHandlerParams) (gri
}
// CheckPartsHandler - check if a file metadata exists.
func (s *storageRESTServer) CheckPartsHandler(p *CheckPartsHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
func (s *storageRESTServer) CheckPartsHandler(p *CheckPartsHandlerParams) (*CheckPartsResp, *grid.RemoteErr) {
if !s.checkID(p.DiskID) {
return grid.NewNPErr(errDiskNotFound)
return nil, grid.NewRemoteErr(errDiskNotFound)
}
volume := p.Volume
filePath := p.FilePath
return grid.NewNPErr(s.getStorage().CheckParts(context.Background(), volume, filePath, p.FI))
resp, err := s.getStorage().CheckParts(context.Background(), volume, filePath, p.FI)
return resp, grid.NewRemoteErr(err)
}
func (s *storageRESTServer) WriteAllHandler(p *WriteAllHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
@ -1097,11 +1099,6 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
}
}
// VerifyFileResp - VerifyFile()'s response.
type VerifyFileResp struct {
Err error
}
// VerifyFileHandler - Verify all part of file for bitrot errors.
func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
@ -1124,13 +1121,15 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
done := keepHTTPResponseAlive(w)
err := s.getStorage().VerifyFile(r.Context(), volume, filePath, fi)
resp, err := s.getStorage().VerifyFile(r.Context(), volume, filePath, fi)
done(nil)
vresp := &VerifyFileResp{}
if err != nil {
vresp.Err = StorageErr(err.Error())
s.writeErrorResponse(w, err)
return
}
encoder.Encode(vresp)
encoder.Encode(resp)
}
func checkDiskFatalErrs(errs []error) error {

13
cmd/xl-storage-disk-id-check.go
View File

@ -487,15 +487,16 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat
})
}
func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCheckParts, volume, path)
if err != nil {
return err
return nil, err
}
defer done(0, &err)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.CheckParts(ctx, volume, path, fi) })
return xioutil.WithDeadline[*CheckPartsResp](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (res *CheckPartsResp, err error) {
return p.storage.CheckParts(ctx, volume, path, fi)
})
}
func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
@ -564,10 +565,10 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string
return errs
}
func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
ctx, done, err := p.TrackDiskHealth(ctx, storageMetricVerifyFile, volume, path)
if err != nil {
return err
return nil, err
}
defer done(0, &err)

74
cmd/xl-storage.go
View File

@ -2312,18 +2312,25 @@ func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string,
}
// CheckParts check if path has necessary parts available.
func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
volumeDir, err := s.getVolDir(volume)
if err != nil {
return err
return nil, err
}
for _, part := range fi.Parts {
err = checkPathLength(pathJoin(volumeDir, path))
if err != nil {
return nil, err
}
resp := CheckPartsResp{
// By default, all results have an unknown status
Results: make([]int, len(fi.Parts)),
}
for i, part := range fi.Parts {
partPath := pathJoin(path, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
filePath := pathJoin(volumeDir, partPath)
if err = checkPathLength(filePath); err != nil {
return err
}
st, err := Lstat(filePath)
if err != nil {
if osIsNotExist(err) {
@ -2331,24 +2338,30 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string,
// Stat a volume entry.
if verr := Access(volumeDir); verr != nil {
if osIsNotExist(verr) {
return errVolumeNotFound
resp.Results[i] = checkPartVolumeNotFound
}
return verr
continue
}
}
}
return osErrToFileErr(err)
if osErrToFileErr(err) == errFileNotFound {
resp.Results[i] = checkPartFileNotFound
}
continue
}
if st.Mode().IsDir() {
return errFileNotFound
resp.Results[i] = checkPartFileNotFound
continue
}
// Check if shard is truncated.
if st.Size() < fi.Erasure.ShardFileSize(part.Size) {
return errFileCorrupt
resp.Results[i] = checkPartFileCorrupt
continue
}
resp.Results[i] = checkPartSuccess
}
return nil
return &resp, nil
}
// deleteFile deletes a file or a directory if its empty unless recursive
@ -2922,42 +2935,43 @@ func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize
return bitrotVerify(diskHealthReader(ctx, file), fi.Size(), partSize, algo, sum, shardSize)
}
func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
volumeDir, err := s.getVolDir(volume)
if err != nil {
return err
return nil, err
}
if !skipAccessChecks(volume) {
// Stat a volume entry.
if err = Access(volumeDir); err != nil {
return convertAccessError(err, errVolumeAccessDenied)
return nil, convertAccessError(err, errVolumeAccessDenied)
}
}
resp := CheckPartsResp{
// By default, the result is unknown per part
Results: make([]int, len(fi.Parts)),
}
erasure := fi.Erasure
for _, part := range fi.Parts {
for i, part := range fi.Parts {
checksumInfo := erasure.GetChecksumInfo(part.Number)
partPath := pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
if err := s.bitrotVerify(ctx, partPath,
err := s.bitrotVerify(ctx, partPath,
erasure.ShardFileSize(part.Size),
checksumInfo.Algorithm,
checksumInfo.Hash, erasure.ShardSize()); err != nil {
if !IsErr(err, []error{
errFileNotFound,
errVolumeNotFound,
errFileCorrupt,
errFileAccessDenied,
errFileVersionNotFound,
}...) {
logger.GetReqInfo(ctx).AppendTags("disk", s.String())
storageLogOnceIf(ctx, err, partPath)
}
return err
checksumInfo.Hash, erasure.ShardSize())
resp.Results[i] = convPartErrToInt(err)
// Only log unknown errors
if resp.Results[i] == checkPartUnknown && err != errFileAccessDenied {
logger.GetReqInfo(ctx).AppendTags("disk", s.String())
storageLogOnceIf(ctx, err, partPath)
}
}
return nil
return &resp, nil
}
// ReadMultiple will read multiple files and send each back as response.

2
internal/grid/handlers.go
View File

@ -112,6 +112,7 @@ const (
HandlerListBuckets
HandlerRenameDataInline
HandlerRenameData2
HandlerCheckParts2
// Add more above here ^^^
// If all handlers are used, the type of Handler can be changed.
@ -192,6 +193,7 @@ var handlerPrefixes = [handlerLast]string{
HandlerListBuckets: peerPrefixS3,
HandlerRenameDataInline: storagePrefix,
HandlerRenameData2: storagePrefix,
HandlerCheckParts2: storagePrefix,
}
const (

11
internal/grid/handlers_string.go
View File

@ -82,14 +82,15 @@ func _() {
_ = x[HandlerListBuckets-71]
_ = x[HandlerRenameDataInline-72]
_ = x[HandlerRenameData2-73]
_ = x[handlerTest-74]
_ = x[handlerTest2-75]
_ = x[handlerLast-76]
_ = x[HandlerCheckParts2-74]
_ = x[handlerTest-75]
_ = x[handlerTest2-76]
_ = x[handlerLast-77]
}
const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2handlerTesthandlerTest2handlerLast"
const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2handlerTesthandlerTest2handlerLast"
var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 938, 949}
var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 937, 949, 960}
func (i HandlerID) String() string {
if i >= HandlerID(len(_HandlerID_index)-1) {