cache DiskInfo at storage layer for performance (#10586)

`mc admin info` on busy setups will not move HDD
heads unnecessarily for repeated calls, provides
a better responsiveness for the call overall.

Bonus change allow listTolerancePerSet be N-1
for good entries, to avoid skipping entries
for some reason one of the disk went offline.

cmd/erasure-common.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"path"
+	"sync"
 
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -53,16 +54,37 @@ func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAP
 
 // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
 // ensures to skip disks if they are not healing and online.
-func (er erasureObjects) getLoadBalancedDisks() (newDisks []StorageAPI) {
+func (er erasureObjects) getLoadBalancedDisks() []StorageAPI {
 	disks := er.getDisks()
 
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	var newDisks []StorageAPI
 	// Based on the random shuffling return back randomized disks.
 	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
-		// Do not consume disks which are being healed.
-		if disks[i-1] != nil && !disks[i-1].Healing() && disks[i-1].IsOnline() {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if disks[i-1] == nil {
+				return
+			}
+			di, err := disks[i-1].DiskInfo(context.Background())
+			if err != nil || di.Healing {
+				// - Do not consume disks which are not reachable
+				//   unformatted or simply not accessible for some reason.
+				//
+				// - Do not consume disks which are being healed
+				//
+				// - Future: skip busy disks
+				return
+			}
+			mu.Lock()
 			newDisks = append(newDisks, disks[i-1])
-		}
+			mu.Unlock()
+		}()
 	}
+	wg.Wait()
 	return newDisks
 }