pick disks which are common maximally used (#10600)

further optimization to ensure that good disks
are always used for listing, other than healing
we only use disks that are maximally used.

cmd/disk-cache-backend.go

@@ -194,7 +194,7 @@ func (c *diskCache) diskUsageLow() bool {
 		logger.LogIf(ctx, err)
 		return false
 	}
-	usedPercent := (di.Total - di.Free) * 100 / di.Total
+	usedPercent := (di.Used / di.Total) * 100
 	low := int(usedPercent) < gcStopPct
 	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
 	if low {
@@ -218,7 +218,7 @@ func (c *diskCache) diskSpaceAvailable(size int64) bool {
 		logger.Info("diskCache: Received 0 total disk size")
 		return false
 	}
-	usedPercent := float64(di.Total-di.Free) * 100 / float64(di.Total)
+	usedPercent := float64(di.Used) * 100 / float64(di.Total)
 	if usedPercent >= float64(gcTriggerPct) {
 		atomic.StoreInt32(&c.stats.UsageState, 1)
 		c.queueGC()
@@ -226,7 +226,7 @@ func (c *diskCache) diskSpaceAvailable(size int64) bool {
 	atomic.StoreUint64(&c.stats.UsagePercent, uint64(usedPercent))
 
 	// Recalculate percentage with provided size added.
-	usedPercent = float64(di.Total-di.Free+uint64(size)) * 100 / float64(di.Total)
+	usedPercent = float64(di.Used+uint64(size)) * 100 / float64(di.Total)
 
 	return usedPercent < float64(c.quotaPct)
 }

cmd/erasure-common.go

@@ -59,7 +59,7 @@ func (er erasureObjects) getLoadBalancedDisks() []StorageAPI {
 
 	var wg sync.WaitGroup
 	var mu sync.Mutex
-	var newDisks []StorageAPI
+	var newDisks = map[uint64][]StorageAPI{}
 	// Based on the random shuffling return back randomized disks.
 	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
 		i := i
@@ -79,13 +79,24 @@ func (er erasureObjects) getLoadBalancedDisks() []StorageAPI {
 				// - Future: skip busy disks
 				return
 			}
+
 			mu.Lock()
-			newDisks = append(newDisks, disks[i-1])
+			// Capture disks usage wise
+			newDisks[di.Used] = append(newDisks[di.Used], disks[i-1])
 			mu.Unlock()
 		}()
 	}
 	wg.Wait()
-	return newDisks
+
+	var max uint64
+	for k := range newDisks {
+		if k > max {
+			max = k
+		}
+	}
+
+	// Return disks which have maximum disk usage common.
+	return newDisks[max]
 }
 
 // This function does the following check, suppose

cmd/erasure-zones.go

@@ -662,7 +662,7 @@ func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix,
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs,
 			zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.listTolerancePerSet, false))
-		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-1)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
 	}
 
 	var objInfos []ObjectInfo
@@ -784,7 +784,7 @@ func (z *erasureZones) listObjectsSplunk(ctx context.Context, bucket, prefix, ma
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-1)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
 	}
 
 	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
@@ -876,7 +876,7 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker,
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-1)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
 	}
 
 	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
@@ -1278,7 +1278,7 @@ func (z *erasureZones) listObjectVersions(ctx context.Context, bucket, prefix, m
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-1)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
 	}
 
 	entries := mergeZonesEntriesVersionsCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)

cmd/fs-v1.go

@@ -216,12 +216,11 @@ func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) (StorageInfo, []er
 	if err != nil {
 		return StorageInfo{}, []error{err}
 	}
-	used := di.Total - di.Free
 	storageInfo := StorageInfo{
 		Disks: []madmin.Disk{
 			{
 				TotalSpace:     di.Total,
-				UsedSpace:      used,
+				UsedSpace:      di.Used,
 				AvailableSpace: di.Free,
 				DrivePath:      fs.fsPath,
 			},

cmd/server-main.go

@@ -319,6 +319,22 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) {
 		if err != nil {
 			return fmt.Errorf("Unable to list buckets to heal: %w", err)
 		}
+		rquorum := InsufficientReadQuorum{}
+		wquorum := InsufficientWriteQuorum{}
+		for _, bucket := range buckets {
+			if err = newObject.MakeBucketWithLocation(ctx, bucket.Name, BucketOptions{}); err != nil {
+				if errors.As(err, &wquorum) || errors.As(err, &rquorum) {
+					// Return the error upwards for the caller to retry.
+					return fmt.Errorf("Unable to heal bucket: %w", err)
+				}
+				if _, ok := err.(BucketExists); !ok {
+					// ignore any other error and log for investigation.
+					logger.LogIf(ctx, err)
+					continue
+				}
+				// Bucket already exists, nothing that needs to be done.
+			}
+		}
 	} else {
 		buckets, err = newObject.ListBuckets(ctx)
 		if err != nil {

cmd/xl-storage.go

@@ -479,7 +479,7 @@ func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) {
 			}
 			dcinfo.Total = di.Total
 			dcinfo.Free = di.Free
-			dcinfo.Used = di.Total - di.Free
+			dcinfo.Used = di.Used
 			dcinfo.FSType = di.FSType
 
 			diskID, err := s.GetDiskID()

pkg/disk/disk.go

@@ -25,10 +25,8 @@ package disk
 type Info struct {
 	Total  uint64
 	Free   uint64
+	Used   uint64
 	Files  uint64
 	Ffree  uint64
 	FSType string
-
-	// Usage is calculated per tenant.
-	Usage uint64
 }

pkg/disk/stat_bsd.go

@@ -19,6 +19,7 @@
 package disk
 
 import (
+	"fmt"
 	"syscall"
 )
 
@@ -37,5 +38,9 @@ func GetInfo(path string) (info Info, err error) {
 		Ffree:  uint64(s.Ffree),
 		FSType: getFSType(s.Fstypename[:]),
 	}
+	if info.Free > info.Total {
+		return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
+	}
+	info.Used = info.Total - info.Free
 	return info, nil
 }

pkg/disk/stat_linux.go

@@ -45,5 +45,6 @@ func GetInfo(path string) (info Info, err error) {
 	if info.Free > info.Total {
 		return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
 	}
+	info.Used = info.Total - info.Free
 	return info, nil
 }

pkg/disk/stat_linux_32bit.go

@@ -76,5 +76,6 @@ func GetInfo(path string) (info Info, err error) {
 	if info.Free > info.Total {
 		return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
 	}
+	info.Used = info.Total - info.Free
 	return info, nil
 }

pkg/disk/stat_linux_s390x.go

@@ -76,5 +76,6 @@ func GetInfo(path string) (info Info, err error) {
 	if info.Free > info.Total {
 		return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
 	}
+	info.Used = info.Total - info.Free
 	return info, nil
 }

pkg/disk/stat_netbsd.go

@@ -19,6 +19,8 @@
 package disk
 
 import (
+	"fmt"
+
 	"golang.org/x/sys/unix"
 )
 
@@ -36,5 +38,9 @@ func GetInfo(path string) (info Info, err error) {
 		Ffree:  uint64(s.Ffree),
 		FSType: string(s.Fstypename[:]),
 	}
+	if info.Free > info.Total {
+		return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
+	}
+	info.Used = info.Total - info.Free
 	return info, nil
 }

pkg/disk/stat_openbsd.go

@@ -19,6 +19,7 @@
 package disk
 
 import (
+	"fmt"
 	"syscall"
 )
 
@@ -37,5 +38,9 @@ func GetInfo(path string) (info Info, err error) {
 		Ffree:  uint64(s.F_ffree),
 		FSType: getFSType(s.F_fstypename[:]),
 	}
+	if info.Free > info.Total {
+		return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
+	}
+	info.Used = info.Total - info.Free
 	return info, nil
 }

pkg/disk/stat_solaris.go

@@ -19,6 +19,8 @@
 package disk
 
 import (
+	"fmt"
+
 	"golang.org/x/sys/unix"
 )
 
@@ -36,5 +38,9 @@ func GetInfo(path string) (info Info, err error) {
 		Ffree:  uint64(s.Ffree),
 		FSType: getFSType(s.Fstr[:]),
 	}
+	if info.Free > info.Total {
+		return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
+	}
+	info.Used = info.Total - info.Free
 	return info, nil
 }

pkg/disk/stat_windows.go

@@ -19,6 +19,7 @@
 package disk
 
 import (
+	"fmt"
 	"os"
 	"syscall"
 	"unsafe"
@@ -90,5 +91,9 @@ func GetInfo(path string) (info Info, err error) {
 	info.Files = uint64(lpTotalNumberOfClusters)
 	info.Ffree = uint64(lpNumberOfFreeClusters)
 
+	if info.Free > info.Total {
+		return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
+	}
+	info.Used = info.Total - info.Free
 	return info, nil
 }