diff --git a/cmd/background-newdisks-heal-ops.go b/cmd/background-newdisks-heal-ops.go index 0b3380954..f7e6aee03 100644 --- a/cmd/background-newdisks-heal-ops.go +++ b/cmd/background-newdisks-heal-ops.go @@ -118,6 +118,7 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) { // 2. Only the node hosting the disk is responsible to perform the heal func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *healSequence) { // Perform automatic disk healing when a disk is replaced locally. +wait: for { select { case <-ctx.Done(): @@ -176,6 +177,26 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq * for _, disk := range disks { logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1)) + // So someone changed the drives underneath, healing tracker missing. + if !disk.Healing() { + logger.Info("Healing tracker missing on '%s', disk was swapped again on %s zone", disk, humanize.Ordinal(i+1)) + diskID, err := disk.GetDiskID() + if err != nil { + logger.LogIf(ctx, err) + // reading format.json failed or not found, proceed to look + // for new disks to be healed again, we cannot proceed further. + goto wait + } + + if err := saveHealingTracker(disk, diskID); err != nil { + logger.LogIf(ctx, err) + // Unable to write healing tracker, permission denied or some + // other unexpected error occurred. Proceed to look for new + // disks to be healed again, we cannot proceed further. + goto wait + } + } + lbDisks := z.serverSets[i].sets[setIndex].getOnlineDisks() if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil { logger.LogIf(ctx, err) diff --git a/cmd/format-erasure.go b/cmd/format-erasure.go index 5e96b20d0..bce86e914 100644 --- a/cmd/format-erasure.go +++ b/cmd/format-erasure.go @@ -339,6 +339,19 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur return formats, g.Wait() } +func saveHealingTracker(disk StorageAPI, diskID string) error { + htracker := healingTracker{ + ID: diskID, + } + htrackerBytes, err := htracker.MarshalMsg(nil) + if err != nil { + return err + } + return disk.WriteAll(context.TODO(), minioMetaBucket, + pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename), + htrackerBytes) +} + func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) error { if disk == nil || format == nil { return errDiskNotFound @@ -373,16 +386,7 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro disk.SetDiskID(diskID) if heal { - htracker := healingTracker{ - ID: diskID, - } - htrackerBytes, err := htracker.MarshalMsg(nil) - if err != nil { - return err - } - return disk.WriteAll(context.TODO(), minioMetaBucket, - pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename), - htrackerBytes) + return saveHealingTracker(disk, diskID) } return nil }