From 789cbc6fb28dabfd0fbd352d4ed204c95d7dccb0 Mon Sep 17 00:00:00 2001 From: Anis Eleuch Date: Mon, 10 Jun 2024 16:51:27 +0100 Subject: [PATCH] heal: Dangling check to evaluate object parts separately (#19797) --- cmd/erasure-healing-common.go | 124 +++++++++++++++++++------ cmd/erasure-healing-common_test.go | 22 +++-- cmd/erasure-healing.go | 128 +++++++++++++++----------- cmd/erasure-healing_test.go | 69 ++++++++++---- cmd/erasure-object.go | 22 +++-- cmd/naughty-disk_test.go | 8 +- cmd/storage-datatypes.go | 17 ++++ cmd/storage-datatypes_gen.go | 139 +++++++++++++++++++++++++++++ cmd/storage-datatypes_gen_test.go | 113 +++++++++++++++++++++++ cmd/storage-interface.go | 4 +- cmd/storage-rest-client.go | 24 ++--- cmd/storage-rest-common.go | 2 +- cmd/storage-rest-server.go | 25 +++--- cmd/xl-storage-disk-id-check.go | 13 +-- cmd/xl-storage.go | 74 ++++++++------- internal/grid/handlers.go | 2 + internal/grid/handlers_string.go | 11 +-- 17 files changed, 614 insertions(+), 183 deletions(-) diff --git a/cmd/erasure-healing-common.go b/cmd/erasure-healing-common.go index aee231ef8..3b10ea55e 100644 --- a/cmd/erasure-healing-common.go +++ b/cmd/erasure-healing-common.go @@ -20,6 +20,7 @@ package cmd import ( "bytes" "context" + "slices" "time" "github.com/minio/madmin-go/v3" @@ -253,6 +254,33 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error, return onlineDisks, modTime, "" } +// Convert verify or check parts returned error to integer representation +func convPartErrToInt(err error) int { + err = unwrapAll(err) + switch err { + case nil: + return checkPartSuccess + case errFileNotFound, errFileVersionNotFound: + return checkPartFileNotFound + case errFileCorrupt: + return checkPartFileCorrupt + case errVolumeNotFound: + return checkPartVolumeNotFound + case errDiskNotFound: + return checkPartDiskNotFound + default: + return checkPartUnknown + } +} + +func partNeedsHealing(partErrs []int) bool { + return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess && i != checkPartUnknown }) > -1 +} + +func hasPartErr(partErrs []int) bool { + return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess }) > -1 +} + // disksWithAllParts - This function needs to be called with // []StorageAPI returned by listOnlineDisks. Returns, // @@ -262,10 +290,19 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error, // a not-found error or a hash-mismatch error. func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo, errs []error, latestMeta FileInfo, bucket, object string, - scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time, -) { - availableDisks := make([]StorageAPI, len(onlineDisks)) - dataErrs := make([]error, len(onlineDisks)) + scanMode madmin.HealScanMode, +) (availableDisks []StorageAPI, dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) { + availableDisks = make([]StorageAPI, len(onlineDisks)) + + dataErrsByDisk = make(map[int][]int, len(onlineDisks)) + for i := range onlineDisks { + dataErrsByDisk[i] = make([]int, len(latestMeta.Parts)) + } + + dataErrsByPart = make(map[int][]int, len(latestMeta.Parts)) + for i := range latestMeta.Parts { + dataErrsByPart[i] = make([]int, len(onlineDisks)) + } inconsistent := 0 for i, meta := range partsMetadata { @@ -295,19 +332,21 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad erasureDistributionReliable = false } + metaErrs := make([]error, len(errs)) + for i, onlineDisk := range onlineDisks { if errs[i] != nil { - dataErrs[i] = errs[i] + metaErrs[i] = errs[i] continue } if onlineDisk == OfflineDisk { - dataErrs[i] = errDiskNotFound + metaErrs[i] = errDiskNotFound continue } meta := partsMetadata[i] if !meta.ModTime.Equal(latestMeta.ModTime) || meta.DataDir != latestMeta.DataDir { - dataErrs[i] = errFileCorrupt + metaErrs[i] = errFileCorrupt partsMetadata[i] = FileInfo{} continue } @@ -315,7 +354,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad if erasureDistributionReliable { if !meta.IsValid() { partsMetadata[i] = FileInfo{} - dataErrs[i] = errFileCorrupt + metaErrs[i] = errFileCorrupt continue } @@ -325,46 +364,79 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad // attempt a fix if possible, assuming other entries // might have the right erasure distribution. partsMetadata[i] = FileInfo{} - dataErrs[i] = errFileCorrupt + metaErrs[i] = errFileCorrupt continue } } } + } + + // Copy meta errors to part errors + for i, err := range metaErrs { + if err != nil { + partErr := convPartErrToInt(err) + for p := range latestMeta.Parts { + dataErrsByPart[p][i] = partErr + } + } + } + + for i, onlineDisk := range onlineDisks { + if metaErrs[i] != nil { + continue + } + meta := partsMetadata[i] + + if meta.Deleted || meta.IsRemote() { + continue + } // Always check data, if we got it. if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 { checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number) - dataErrs[i] = bitrotVerify(bytes.NewReader(meta.Data), + verifyErr := bitrotVerify(bytes.NewReader(meta.Data), int64(len(meta.Data)), meta.Erasure.ShardFileSize(meta.Size), checksumInfo.Algorithm, checksumInfo.Hash, meta.Erasure.ShardSize()) - if dataErrs[i] == nil { - // All parts verified, mark it as all data available. - availableDisks[i] = onlineDisk - } else { - // upon errors just make that disk's fileinfo invalid - partsMetadata[i] = FileInfo{} - } + dataErrsByPart[0][i] = convPartErrToInt(verifyErr) continue } + var ( + verifyErr error + verifyResp *CheckPartsResp + ) + meta.DataDir = latestMeta.DataDir switch scanMode { case madmin.HealDeepScan: // disk has a valid xl.meta but may not have all the // parts. This is considered an outdated disk, since // it needs healing too. - if !meta.Deleted && !meta.IsRemote() { - dataErrs[i] = onlineDisk.VerifyFile(ctx, bucket, object, meta) - } - case madmin.HealNormalScan: - if !meta.Deleted && !meta.IsRemote() { - dataErrs[i] = onlineDisk.CheckParts(ctx, bucket, object, meta) - } + verifyResp, verifyErr = onlineDisk.VerifyFile(ctx, bucket, object, meta) + default: + verifyResp, verifyErr = onlineDisk.CheckParts(ctx, bucket, object, meta) } - if dataErrs[i] == nil { + for p := range latestMeta.Parts { + if verifyErr != nil { + dataErrsByPart[p][i] = convPartErrToInt(verifyErr) + } else { + dataErrsByPart[p][i] = verifyResp.Results[p] + } + } + } + + // Build dataErrs by disk from dataErrs by part + for part, disks := range dataErrsByPart { + for disk := range disks { + dataErrsByDisk[disk][part] = dataErrsByPart[part][disk] + } + } + + for i, onlineDisk := range onlineDisks { + if metaErrs[i] == nil && !hasPartErr(dataErrsByDisk[i]) { // All parts verified, mark it as all data available. availableDisks[i] = onlineDisk } else { @@ -373,5 +445,5 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad } } - return availableDisks, dataErrs, timeSentinel + return } diff --git a/cmd/erasure-healing-common_test.go b/cmd/erasure-healing-common_test.go index abcb9ff3d..99b1ca99e 100644 --- a/cmd/erasure-healing-common_test.go +++ b/cmd/erasure-healing-common_test.go @@ -308,9 +308,8 @@ func TestListOnlineDisks(t *testing.T) { t.Fatalf("Expected modTime to be equal to %v but was found to be %v", test.expectedTime, modTime) } - availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata, + availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata, test.errs, fi, bucket, object, madmin.HealDeepScan) - test.errs = newErrs if test._tamperBackend != noTamper { if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil { @@ -491,9 +490,8 @@ func TestListOnlineDisksSmallObjects(t *testing.T) { test.expectedTime, modTime) } - availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata, + availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata, test.errs, fi, bucket, object, madmin.HealDeepScan) - test.errs = newErrs if test._tamperBackend != noTamper { if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil { @@ -554,7 +552,7 @@ func TestDisksWithAllParts(t *testing.T) { erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum) - filteredDisks, errs, _ := disksWithAllParts(ctx, erasureDisks, partsMetadata, + filteredDisks, _, dataErrsPerDisk := disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan) if len(filteredDisks) != len(erasureDisks) { @@ -562,8 +560,8 @@ func TestDisksWithAllParts(t *testing.T) { } for diskIndex, disk := range filteredDisks { - if errs[diskIndex] != nil { - t.Errorf("Unexpected error %s", errs[diskIndex]) + if partNeedsHealing(dataErrsPerDisk[diskIndex]) { + t.Errorf("Unexpected error: %v", dataErrsPerDisk[diskIndex]) } if disk == nil { @@ -634,7 +632,7 @@ func TestDisksWithAllParts(t *testing.T) { } errs = make([]error, len(erasureDisks)) - filteredDisks, errs, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, + filteredDisks, dataErrsPerDisk, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan) if len(filteredDisks) != len(erasureDisks) { @@ -646,15 +644,15 @@ func TestDisksWithAllParts(t *testing.T) { if disk != nil { t.Errorf("Drive not filtered as expected, drive: %d", diskIndex) } - if errs[diskIndex] == nil { - t.Errorf("Expected error not received, driveIndex: %d", diskIndex) + if !partNeedsHealing(dataErrsPerDisk[diskIndex]) { + t.Errorf("Disk expected to be healed, driveIndex: %d", diskIndex) } } else { if disk == nil { t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex) } - if errs[diskIndex] != nil { - t.Errorf("Unexpected error, %s, driveIndex: %d", errs[diskIndex], diskIndex) + if partNeedsHealing(dataErrsPerDisk[diskIndex]) { + t.Errorf("Disk not expected to be healed, driveIndex: %d", diskIndex) } } diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go index 660d232f4..538d8bc81 100644 --- a/cmd/erasure-healing.go +++ b/cmd/erasure-healing.go @@ -32,6 +32,7 @@ import ( "github.com/minio/minio/internal/grid" "github.com/minio/minio/internal/logger" "github.com/minio/pkg/v3/sync/errgroup" + "golang.org/x/exp/slices" ) //go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE @@ -144,36 +145,41 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets return reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum) } +var errLegacyXLMeta = errors.New("legacy XL meta") + +var errOutdatedXLMeta = errors.New("outdated XL meta") + +var errPartMissingOrCorrupt = errors.New("part missing or corrupt") + // Only heal on disks where we are sure that healing is needed. We can expand // this list as and when we figure out more errors can be added to this list safely. -func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo) bool { - switch { - case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound): - return true - case errors.Is(erErr, errFileCorrupt): - return true +func shouldHealObjectOnDisk(erErr error, partsErrs []int, meta FileInfo, latestMeta FileInfo) (bool, error) { + if errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound) || errors.Is(erErr, errFileCorrupt) { + return true, erErr } if erErr == nil { if meta.XLV1 { // Legacy means heal always // always check first. - return true + return true, errLegacyXLMeta + } + if !latestMeta.Equals(meta) { + return true, errOutdatedXLMeta } if !meta.Deleted && !meta.IsRemote() { // If xl.meta was read fine but there may be problem with the part.N files. - if IsErr(dataErr, []error{ - errFileNotFound, - errFileVersionNotFound, - errFileCorrupt, - }...) { - return true + for _, partErr := range partsErrs { + if slices.Contains([]int{ + checkPartFileNotFound, + checkPartFileCorrupt, + }, partErr) { + return true, errPartMissingOrCorrupt + } } } - if !latestMeta.Equals(meta) { - return true - } + return false, nil } - return false + return false, erErr } const ( @@ -332,7 +338,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object // used here for reconstruction. This is done to ensure that // we do not skip drives that have inconsistent metadata to be // skipped from purging when they are stale. - availableDisks, dataErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata, + availableDisks, dataErrsByDisk, dataErrsByPart := disksWithAllParts(ctx, onlineDisks, partsMetadata, errs, latestMeta, bucket, object, scanMode) var erasure Erasure @@ -355,15 +361,20 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object // to be healed. outDatedDisks := make([]StorageAPI, len(storageDisks)) disksToHealCount := 0 - for i, v := range availableDisks { + for i := range availableDisks { + yes, reason := shouldHealObjectOnDisk(errs[i], dataErrsByDisk[i], partsMetadata[i], latestMeta) + if yes { + outDatedDisks[i] = storageDisks[i] + disksToHealCount++ + } + driveState := "" switch { - case v != nil: + case reason == nil: driveState = madmin.DriveStateOk - case errors.Is(errs[i], errDiskNotFound), errors.Is(dataErrs[i], errDiskNotFound): + case IsErr(reason, errDiskNotFound): driveState = madmin.DriveStateOffline - case IsErr(errs[i], errFileNotFound, errFileVersionNotFound, errVolumeNotFound), - IsErr(dataErrs[i], errFileNotFound, errFileVersionNotFound, errVolumeNotFound): + case IsErr(reason, errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errPartMissingOrCorrupt, errOutdatedXLMeta, errLegacyXLMeta): driveState = madmin.DriveStateMissing default: // all remaining cases imply corrupt data/metadata @@ -380,12 +391,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object Endpoint: storageEndpoints[i].String(), State: driveState, }) - - if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) { - outDatedDisks[i] = storageDisks[i] - disksToHealCount++ - continue - } } if isAllNotFound(errs) { @@ -412,7 +417,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks { // Allow for dangling deletes, on versions that have DataDir missing etc. // this would end up restoring the correct readable versions. - m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrs, ObjectOptions{ + m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrsByPart, ObjectOptions{ VersionID: versionID, }) errs = make([]error, len(errs)) @@ -908,35 +913,52 @@ func isObjectDirDangling(errs []error) (ok bool) { return found < notFound && found > 0 } +func danglingMetaErrsCount(cerrs []error) (notFoundCount int, nonActionableCount int) { + for _, readErr := range cerrs { + if readErr == nil { + continue + } + switch { + case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound): + notFoundCount++ + default: + // All other errors are non-actionable + nonActionableCount++ + } + } + return +} + +func danglingPartErrsCount(results []int) (notFoundCount int, nonActionableCount int) { + for _, partResult := range results { + switch partResult { + case checkPartSuccess: + continue + case checkPartFileNotFound: + notFoundCount++ + default: + // All other errors are non-actionable + nonActionableCount++ + } + } + return +} + // Object is considered dangling/corrupted if and only // if total disks - a combination of corrupted and missing // files is lesser than number of data blocks. -func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (validMeta FileInfo, ok bool) { +func isObjectDangling(metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int) (validMeta FileInfo, ok bool) { // We can consider an object data not reliable // when xl.meta is not found in read quorum disks. // or when xl.meta is not readable in read quorum disks. - danglingErrsCount := func(cerrs []error) (int, int) { - var ( - notFoundCount int - nonActionableCount int - ) - for _, readErr := range cerrs { - if readErr == nil { - continue - } - switch { - case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound): - notFoundCount++ - default: - // All other errors are non-actionable - nonActionableCount++ - } - } - return notFoundCount, nonActionableCount - } + notFoundMetaErrs, nonActionableMetaErrs := danglingMetaErrsCount(errs) - notFoundMetaErrs, nonActionableMetaErrs := danglingErrsCount(errs) - notFoundPartsErrs, nonActionablePartsErrs := danglingErrsCount(dataErrs) + notFoundPartsErrs, nonActionablePartsErrs := 0, 0 + for _, dataErrs := range dataErrsByPart { + if nf, na := danglingPartErrsCount(dataErrs); nf > notFoundPartsErrs { + notFoundPartsErrs, nonActionablePartsErrs = nf, na + } + } for _, m := range metaArr { if m.IsValid() { @@ -948,7 +970,7 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid if !validMeta.IsValid() { // validMeta is invalid because all xl.meta is missing apparently // we should figure out if dataDirs are also missing > dataBlocks. - dataBlocks := (len(dataErrs) + 1) / 2 + dataBlocks := (len(metaArr) + 1) / 2 if notFoundPartsErrs > dataBlocks { // Not using parity to ensure that we do not delete // any valid content, if any is recoverable. But if diff --git a/cmd/erasure-healing_test.go b/cmd/erasure-healing_test.go index c4abdf65c..5a95e84a7 100644 --- a/cmd/erasure-healing_test.go +++ b/cmd/erasure-healing_test.go @@ -49,7 +49,7 @@ func TestIsObjectDangling(t *testing.T) { name string metaArr []FileInfo errs []error - dataErrs []error + dataErrs map[int][]int expectedMeta FileInfo expectedDangling bool }{ @@ -165,11 +165,8 @@ func TestIsObjectDangling(t *testing.T) { nil, nil, }, - dataErrs: []error{ - errFileCorrupt, - errFileNotFound, - nil, - errFileCorrupt, + dataErrs: map[int][]int{ + 0: {checkPartFileCorrupt, checkPartFileNotFound, checkPartSuccess, checkPartFileCorrupt}, }, expectedMeta: fi, expectedDangling: false, @@ -188,11 +185,8 @@ func TestIsObjectDangling(t *testing.T) { errFileNotFound, nil, }, - dataErrs: []error{ - errFileNotFound, - errFileCorrupt, - nil, - nil, + dataErrs: map[int][]int{ + 0: {checkPartFileNotFound, checkPartFileCorrupt, checkPartSuccess, checkPartSuccess}, }, expectedMeta: fi, expectedDangling: false, @@ -247,15 +241,58 @@ func TestIsObjectDangling(t *testing.T) { nil, nil, }, - dataErrs: []error{ - errFileNotFound, - errFileNotFound, - nil, - errFileNotFound, + dataErrs: map[int][]int{ + 0: {checkPartFileNotFound, checkPartFileNotFound, checkPartSuccess, checkPartFileNotFound}, }, expectedMeta: fi, expectedDangling: true, }, + { + name: "FileInfoDecided-case4-(missing data-dir for part 2)", + metaArr: []FileInfo{ + {}, + {}, + {}, + fi, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + nil, + nil, + }, + dataErrs: map[int][]int{ + 0: {checkPartSuccess, checkPartSuccess, checkPartSuccess, checkPartSuccess}, + 1: {checkPartSuccess, checkPartFileNotFound, checkPartFileNotFound, checkPartFileNotFound}, + }, + expectedMeta: fi, + expectedDangling: true, + }, + + { + name: "FileInfoDecided-case4-(enough data-dir existing for each part)", + metaArr: []FileInfo{ + {}, + {}, + {}, + fi, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + nil, + nil, + }, + dataErrs: map[int][]int{ + 0: {checkPartFileNotFound, checkPartSuccess, checkPartSuccess, checkPartSuccess}, + 1: {checkPartSuccess, checkPartFileNotFound, checkPartSuccess, checkPartSuccess}, + 2: {checkPartSuccess, checkPartSuccess, checkPartFileNotFound, checkPartSuccess}, + 3: {checkPartSuccess, checkPartSuccess, checkPartSuccess, checkPartFileNotFound}, + }, + expectedMeta: fi, + expectedDangling: false, + }, + // Add new cases as seen } for _, testCase := range testCases { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 2540029c3..ba137f6ba 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -484,8 +484,8 @@ func joinErrs(errs []error) []string { return s } -func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) { - m, ok := isObjectDangling(metaArr, errs, dataErrs) +func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int, opts ObjectOptions) (FileInfo, error) { + m, ok := isObjectDangling(metaArr, errs, dataErrsByPart) if !ok { // We only come here if we cannot figure out if the object // can be deleted safely, in such a scenario return ReadQuorum error. @@ -495,7 +495,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st tags["set"] = er.setIndex tags["pool"] = er.poolIndex tags["merrs"] = joinErrs(errs) - tags["derrs"] = joinErrs(dataErrs) + tags["derrs"] = dataErrsByPart if m.IsValid() { tags["size"] = m.Size tags["mtime"] = m.ModTime.Format(http.TimeFormat) @@ -509,8 +509,20 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st // count the number of offline disks offline := 0 - for i := 0; i < max(len(errs), len(dataErrs)); i++ { - if i < len(errs) && errors.Is(errs[i], errDiskNotFound) || i < len(dataErrs) && errors.Is(dataErrs[i], errDiskNotFound) { + for i := 0; i < len(errs); i++ { + var found bool + switch { + case errors.Is(errs[i], errDiskNotFound): + found = true + default: + for p := range dataErrsByPart { + if dataErrsByPart[p][i] == checkPartDiskNotFound { + found = true + break + } + } + } + if found { offline++ } } diff --git a/cmd/naughty-disk_test.go b/cmd/naughty-disk_test.go index 86c577eca..692b7b80d 100644 --- a/cmd/naughty-disk_test.go +++ b/cmd/naughty-disk_test.go @@ -215,9 +215,9 @@ func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVol return d.disk.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) } -func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) { +func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) { if err := d.calcError(); err != nil { - return err + return nil, err } return d.disk.CheckParts(ctx, volume, path, fi) } @@ -289,9 +289,9 @@ func (d *naughtyDisk) ReadXL(ctx context.Context, volume string, path string, re return d.disk.ReadXL(ctx, volume, path, readData) } -func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error { +func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) { if err := d.calcError(); err != nil { - return err + return nil, err } return d.disk.VerifyFile(ctx, volume, path, fi) } diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go index 4f1accffc..eb74d9e77 100644 --- a/cmd/storage-datatypes.go +++ b/cmd/storage-datatypes.go @@ -502,6 +502,23 @@ type RenameDataResp struct { OldDataDir string // contains '', it is designed to be passed as value to Delete(bucket, pathJoin(object, dataDir)) } +const ( + checkPartUnknown int = iota + + // Changing the order can cause a data loss + // when running two nodes with incompatible versions + checkPartSuccess + checkPartDiskNotFound + checkPartVolumeNotFound + checkPartFileNotFound + checkPartFileCorrupt +) + +// CheckPartsResp is a response of the storage CheckParts and VerifyFile APIs +type CheckPartsResp struct { + Results []int +} + // LocalDiskIDs - GetLocalIDs response. type LocalDiskIDs struct { IDs []string diff --git a/cmd/storage-datatypes_gen.go b/cmd/storage-datatypes_gen.go index e9d584f2c..f70f1b186 100644 --- a/cmd/storage-datatypes_gen.go +++ b/cmd/storage-datatypes_gen.go @@ -273,6 +273,145 @@ func (z *CheckPartsHandlerParams) Msgsize() (s int) { return } +// DecodeMsg implements msgp.Decodable +func (z *CheckPartsResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Results": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Results") + return + } + if cap(z.Results) >= int(zb0002) { + z.Results = (z.Results)[:zb0002] + } else { + z.Results = make([]int, zb0002) + } + for za0001 := range z.Results { + z.Results[za0001], err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Results", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *CheckPartsResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "Results" + err = en.Append(0x81, 0xa7, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Results))) + if err != nil { + err = msgp.WrapError(err, "Results") + return + } + for za0001 := range z.Results { + err = en.WriteInt(z.Results[za0001]) + if err != nil { + err = msgp.WrapError(err, "Results", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *CheckPartsResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "Results" + o = append(o, 0x81, 0xa7, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Results))) + for za0001 := range z.Results { + o = msgp.AppendInt(o, z.Results[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CheckPartsResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Results": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Results") + return + } + if cap(z.Results) >= int(zb0002) { + z.Results = (z.Results)[:zb0002] + } else { + z.Results = make([]int, zb0002) + } + for za0001 := range z.Results { + z.Results[za0001], bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Results", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *CheckPartsResp) Msgsize() (s int) { + s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.Results) * (msgp.IntSize)) + return +} + // DecodeMsg implements msgp.Decodable func (z *DeleteFileHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte diff --git a/cmd/storage-datatypes_gen_test.go b/cmd/storage-datatypes_gen_test.go index a801cfd34..fe7d592a7 100644 --- a/cmd/storage-datatypes_gen_test.go +++ b/cmd/storage-datatypes_gen_test.go @@ -235,6 +235,119 @@ func BenchmarkDecodeCheckPartsHandlerParams(b *testing.B) { } } +func TestMarshalUnmarshalCheckPartsResp(t *testing.T) { + v := CheckPartsResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeCheckPartsResp(t *testing.T) { + v := CheckPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeCheckPartsResp Msgsize() is inaccurate") + } + + vn := CheckPartsResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalDeleteFileHandlerParams(t *testing.T) { v := DeleteFileHandlerParams{} bts, err := v.MarshalMsg(nil) diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go index e8117b99c..04f77da7d 100644 --- a/cmd/storage-interface.go +++ b/cmd/storage-interface.go @@ -94,9 +94,9 @@ type StorageAPI interface { CreateFile(ctx context.Context, origvolume, olume, path string, size int64, reader io.Reader) error ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error - CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error + CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) Delete(ctx context.Context, volume string, path string, opts DeleteOptions) (err error) - VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error + VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error CleanAbandonedData(ctx context.Context, volume string, path string) error diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index f6d0352e2..06bcadc19 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -449,14 +449,18 @@ func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, pa } // CheckParts - stat all file parts. -func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error { - _, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{ +func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) { + var resp *CheckPartsResp + resp, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{ DiskID: *client.diskID.Load(), Volume: volume, FilePath: path, FI: fi, }) - return toStorageErr(err) + if err != nil { + return nil, err + } + return resp, nil } // RenameData - rename source path to destination path atomically, metadata and data file. @@ -748,33 +752,33 @@ func (client *storageRESTClient) RenameFile(ctx context.Context, srcVolume, srcP return toStorageErr(err) } -func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error { +func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) var reader bytes.Buffer if err := msgp.Encode(&reader, &fi); err != nil { - return err + return nil, err } respBody, err := client.call(ctx, storageRESTMethodVerifyFile, values, &reader, -1) defer xhttp.DrainBody(respBody) if err != nil { - return err + return nil, err } respReader, err := waitForHTTPResponse(respBody) if err != nil { - return toStorageErr(err) + return nil, toStorageErr(err) } - verifyResp := &VerifyFileResp{} + verifyResp := &CheckPartsResp{} if err = gob.NewDecoder(respReader).Decode(verifyResp); err != nil { - return toStorageErr(err) + return nil, toStorageErr(err) } - return toStorageErr(verifyResp.Err) + return verifyResp, nil } func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { diff --git a/cmd/storage-rest-common.go b/cmd/storage-rest-common.go index 32fe56c1a..e87e54b92 100644 --- a/cmd/storage-rest-common.go +++ b/cmd/storage-rest-common.go @@ -20,7 +20,7 @@ package cmd //go:generate msgp -file $GOFILE -unexported const ( - storageRESTVersion = "v57" // Remove TotalTokens from DiskMetrics + storageRESTVersion = "v58" // Change VerifyFile signature storageRESTVersionPrefix = SlashSeparator + storageRESTVersion storageRESTPrefix = minioReservedBucketPath + "/storage" ) diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index 7e0b24d0d..98681e224 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -58,7 +58,7 @@ type storageRESTServer struct { } var ( - storageCheckPartsRPC = grid.NewSingleHandler[*CheckPartsHandlerParams, grid.NoPayload](grid.HandlerCheckParts, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, grid.NewNoPayload) + storageCheckPartsRPC = grid.NewSingleHandler[*CheckPartsHandlerParams, *CheckPartsResp](grid.HandlerCheckParts2, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, func() *CheckPartsResp { return &CheckPartsResp{} }) storageDeleteFileRPC = grid.NewSingleHandler[*DeleteFileHandlerParams, grid.NoPayload](grid.HandlerDeleteFile, func() *DeleteFileHandlerParams { return &DeleteFileHandlerParams{} }, grid.NewNoPayload).AllowCallRequestPool(true) storageDeleteVersionRPC = grid.NewSingleHandler[*DeleteVersionHandlerParams, grid.NoPayload](grid.HandlerDeleteVersion, func() *DeleteVersionHandlerParams { return &DeleteVersionHandlerParams{} }, grid.NewNoPayload) storageDiskInfoRPC = grid.NewSingleHandler[*DiskInfoOptions, *DiskInfo](grid.HandlerDiskInfo, func() *DiskInfoOptions { return &DiskInfoOptions{} }, func() *DiskInfo { return &DiskInfo{} }).WithSharedResponse().AllowCallRequestPool(true) @@ -439,13 +439,15 @@ func (s *storageRESTServer) UpdateMetadataHandler(p *MetadataHandlerParams) (gri } // CheckPartsHandler - check if a file metadata exists. -func (s *storageRESTServer) CheckPartsHandler(p *CheckPartsHandlerParams) (grid.NoPayload, *grid.RemoteErr) { +func (s *storageRESTServer) CheckPartsHandler(p *CheckPartsHandlerParams) (*CheckPartsResp, *grid.RemoteErr) { if !s.checkID(p.DiskID) { - return grid.NewNPErr(errDiskNotFound) + return nil, grid.NewRemoteErr(errDiskNotFound) } volume := p.Volume filePath := p.FilePath - return grid.NewNPErr(s.getStorage().CheckParts(context.Background(), volume, filePath, p.FI)) + + resp, err := s.getStorage().CheckParts(context.Background(), volume, filePath, p.FI) + return resp, grid.NewRemoteErr(err) } func (s *storageRESTServer) WriteAllHandler(p *WriteAllHandlerParams) (grid.NoPayload, *grid.RemoteErr) { @@ -1097,11 +1099,6 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error { } } -// VerifyFileResp - VerifyFile()'s response. -type VerifyFileResp struct { - Err error -} - // VerifyFileHandler - Verify all part of file for bitrot errors. func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Request) { if !s.IsValid(w, r) { @@ -1124,13 +1121,15 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req setEventStreamHeaders(w) encoder := gob.NewEncoder(w) done := keepHTTPResponseAlive(w) - err := s.getStorage().VerifyFile(r.Context(), volume, filePath, fi) + resp, err := s.getStorage().VerifyFile(r.Context(), volume, filePath, fi) done(nil) - vresp := &VerifyFileResp{} + if err != nil { - vresp.Err = StorageErr(err.Error()) + s.writeErrorResponse(w, err) + return } - encoder.Encode(vresp) + + encoder.Encode(resp) } func checkDiskFatalErrs(errs []error) error { diff --git a/cmd/xl-storage-disk-id-check.go b/cmd/xl-storage-disk-id-check.go index ad60b556a..97b896a7a 100644 --- a/cmd/xl-storage-disk-id-check.go +++ b/cmd/xl-storage-disk-id-check.go @@ -487,15 +487,16 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat }) } -func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) { +func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) { ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCheckParts, volume, path) if err != nil { - return err + return nil, err } defer done(0, &err) - w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) - return w.Run(func() error { return p.storage.CheckParts(ctx, volume, path, fi) }) + return xioutil.WithDeadline[*CheckPartsResp](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (res *CheckPartsResp, err error) { + return p.storage.CheckParts(ctx, volume, path, fi) + }) } func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) { @@ -564,10 +565,10 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string return errs } -func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) { +func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) { ctx, done, err := p.TrackDiskHealth(ctx, storageMetricVerifyFile, volume, path) if err != nil { - return err + return nil, err } defer done(0, &err) diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 0ae54541e..c7746bfdf 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -2312,18 +2312,25 @@ func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string, } // CheckParts check if path has necessary parts available. -func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error { +func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) { volumeDir, err := s.getVolDir(volume) if err != nil { - return err + return nil, err } - for _, part := range fi.Parts { + err = checkPathLength(pathJoin(volumeDir, path)) + if err != nil { + return nil, err + } + + resp := CheckPartsResp{ + // By default, all results have an unknown status + Results: make([]int, len(fi.Parts)), + } + + for i, part := range fi.Parts { partPath := pathJoin(path, fi.DataDir, fmt.Sprintf("part.%d", part.Number)) filePath := pathJoin(volumeDir, partPath) - if err = checkPathLength(filePath); err != nil { - return err - } st, err := Lstat(filePath) if err != nil { if osIsNotExist(err) { @@ -2331,24 +2338,30 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, // Stat a volume entry. if verr := Access(volumeDir); verr != nil { if osIsNotExist(verr) { - return errVolumeNotFound + resp.Results[i] = checkPartVolumeNotFound } - return verr + continue } } } - return osErrToFileErr(err) + if osErrToFileErr(err) == errFileNotFound { + resp.Results[i] = checkPartFileNotFound + } + continue } if st.Mode().IsDir() { - return errFileNotFound + resp.Results[i] = checkPartFileNotFound + continue } // Check if shard is truncated. if st.Size() < fi.Erasure.ShardFileSize(part.Size) { - return errFileCorrupt + resp.Results[i] = checkPartFileCorrupt + continue } + resp.Results[i] = checkPartSuccess } - return nil + return &resp, nil } // deleteFile deletes a file or a directory if its empty unless recursive @@ -2922,42 +2935,43 @@ func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize return bitrotVerify(diskHealthReader(ctx, file), fi.Size(), partSize, algo, sum, shardSize) } -func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) { +func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) { volumeDir, err := s.getVolDir(volume) if err != nil { - return err + return nil, err } if !skipAccessChecks(volume) { // Stat a volume entry. if err = Access(volumeDir); err != nil { - return convertAccessError(err, errVolumeAccessDenied) + return nil, convertAccessError(err, errVolumeAccessDenied) } } + resp := CheckPartsResp{ + // By default, the result is unknown per part + Results: make([]int, len(fi.Parts)), + } + erasure := fi.Erasure - for _, part := range fi.Parts { + for i, part := range fi.Parts { checksumInfo := erasure.GetChecksumInfo(part.Number) partPath := pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", part.Number)) - if err := s.bitrotVerify(ctx, partPath, + err := s.bitrotVerify(ctx, partPath, erasure.ShardFileSize(part.Size), checksumInfo.Algorithm, - checksumInfo.Hash, erasure.ShardSize()); err != nil { - if !IsErr(err, []error{ - errFileNotFound, - errVolumeNotFound, - errFileCorrupt, - errFileAccessDenied, - errFileVersionNotFound, - }...) { - logger.GetReqInfo(ctx).AppendTags("disk", s.String()) - storageLogOnceIf(ctx, err, partPath) - } - return err + checksumInfo.Hash, erasure.ShardSize()) + + resp.Results[i] = convPartErrToInt(err) + + // Only log unknown errors + if resp.Results[i] == checkPartUnknown && err != errFileAccessDenied { + logger.GetReqInfo(ctx).AppendTags("disk", s.String()) + storageLogOnceIf(ctx, err, partPath) } } - return nil + return &resp, nil } // ReadMultiple will read multiple files and send each back as response. diff --git a/internal/grid/handlers.go b/internal/grid/handlers.go index 0f5411ee1..9569e0f38 100644 --- a/internal/grid/handlers.go +++ b/internal/grid/handlers.go @@ -112,6 +112,7 @@ const ( HandlerListBuckets HandlerRenameDataInline HandlerRenameData2 + HandlerCheckParts2 // Add more above here ^^^ // If all handlers are used, the type of Handler can be changed. @@ -192,6 +193,7 @@ var handlerPrefixes = [handlerLast]string{ HandlerListBuckets: peerPrefixS3, HandlerRenameDataInline: storagePrefix, HandlerRenameData2: storagePrefix, + HandlerCheckParts2: storagePrefix, } const ( diff --git a/internal/grid/handlers_string.go b/internal/grid/handlers_string.go index 07c8d810c..51ed08c9c 100644 --- a/internal/grid/handlers_string.go +++ b/internal/grid/handlers_string.go @@ -82,14 +82,15 @@ func _() { _ = x[HandlerListBuckets-71] _ = x[HandlerRenameDataInline-72] _ = x[HandlerRenameData2-73] - _ = x[handlerTest-74] - _ = x[handlerTest2-75] - _ = x[handlerLast-76] + _ = x[HandlerCheckParts2-74] + _ = x[handlerTest-75] + _ = x[handlerTest2-76] + _ = x[handlerLast-77] } -const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2handlerTesthandlerTest2handlerLast" +const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2handlerTesthandlerTest2handlerLast" -var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 938, 949} +var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 937, 949, 960} func (i HandlerID) String() string { if i >= HandlerID(len(_HandlerID_index)-1) {