Heal buckets at node level (#18504)

cmd/erasure-server-pool.go

@@ -1923,8 +1923,55 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
 	hopts.Recreate = false
 	defer z.HealObject(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile), "", hopts)
 
+	type DiskStat struct {
+		VolInfos []VolInfo
+		Errs     []error
+	}
+
 	for _, pool := range z.serverPools {
-		result, err := pool.HealBucket(ctx, bucket, opts)
+		// map of node wise disk stats
+		diskStats := make(map[string]DiskStat)
+		for _, set := range pool.sets {
+			vis := []VolInfo{}
+			errs := []error{}
+			for _, disk := range set.getDisks() {
+				if disk == OfflineDisk {
+					continue
+				}
+				vi, err := disk.StatVol(ctx, bucket)
+				hostName := disk.Hostname()
+				if disk.IsLocal() {
+					hostName = "local"
+				}
+				ds, ok := diskStats[hostName]
+				if !ok {
+					newds := DiskStat{
+						VolInfos: vis,
+						Errs:     errs,
+					}
+					diskStats[hostName] = newds
+				} else {
+					ds.VolInfos = append(ds.VolInfos, vi)
+					ds.Errs = append(ds.Errs, err)
+					diskStats[hostName] = ds
+				}
+			}
+		}
+		nodeCount := len(diskStats)
+		bktNotFoundCount := 0
+		for _, ds := range diskStats {
+			if isAllBucketsNotFound(ds.Errs) {
+				bktNotFoundCount++
+			}
+		}
+		// if the bucket if not found on more than hslf the no of nodes, its dagling
+		if bktNotFoundCount > nodeCount/2 {
+			opts.Remove = true
+		} else {
+			opts.Recreate = true
+		}
+
+		result, err := z.s3Peer.HealBucket(ctx, bucket, opts)
 		if err != nil {
 			if _, ok := err.(BucketNotFound); ok {
 				continue