metrics: Use StorageInfo() instead to have consistent info (#9006)

Metrics used to have its own code to calculate offline disks.
StorageInfo() was avoided because it is an expensive operation
by sending calls to all nodes.

To make metrics & server info share the same code, a new
argument `local` is added to StorageInfo() so it will only
query local disks when needed.

Metrics now calls StorageInfo() as server info handler does
but with the local flag set to false.

Co-authored-by: Praveen raj Mani <praveen@minio.io>
Co-authored-by: Harshavardhana <harsha@minio.io>

cmd/admin-handlers.go

@@ -276,7 +276,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	// Marshal API response
 	jsonBytes, err := json.Marshal(storageInfo)
@@ -897,7 +897,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// find number of disks in the setup
-	info := objectAPI.StorageInfo(ctx)
+	info := objectAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
 
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
@@ -1410,7 +1410,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	notifyTarget := fetchLambdaInfo(cfg)
 
 	// Fetching the Storage information
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	var OnDisks int
 	var OffDisks int

cmd/background-heal-ops.go

@@ -127,7 +127,7 @@ func startBackgroundHealing() {
 
 	// Launch the background healer sequence to track
 	// background healing operations
-	info := objAPI.StorageInfo(ctx)
+	info := objAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
 	nh := newBgHealSequence(numDisks)
 	globalBackgroundHealState.LaunchNewHealSequence(nh)

cmd/fs-v1.go

@@ -202,7 +202,7 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo - returns underlying storage statistics.
-func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
+func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo {
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {

cmd/gateway/azure/gateway-azure.go

@@ -481,7 +481,7 @@ func (a *azureObjects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo - Not relevant to Azure backend.
-func (a *azureObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, a.httpClient, a.endpoint)
 	return si

cmd/gateway/b2/gateway-b2.go

@@ -218,7 +218,7 @@ func (l *b2Objects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo is not relevant to B2 backend.
-func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (l *b2Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://api.backblazeb2.com/b2api/v1")
 	return si

cmd/gateway/gcs/gateway-gcs.go

@@ -412,7 +412,7 @@ func (l *gcsGateway) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo - Not relevant to GCS backend.
-func (l *gcsGateway) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com")
 	return si

cmd/gateway/hdfs/gateway-hdfs.go

@@ -203,7 +203,7 @@ func (n *hdfsObjects) Shutdown(ctx context.Context) error {
 	return n.clnt.Close()
 }
 
-func (n *hdfsObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
+func (n *hdfsObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
 	fsInfo, err := n.clnt.StatFs()
 	if err != nil {
 		return minio.StorageInfo{}

cmd/gateway/nas/gateway-nas.go

@@ -106,8 +106,8 @@ func (n *nasObjects) IsListenBucketSupported() bool {
 	return false
 }
 
-func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
-	sinfo := n.ObjectLayer.StorageInfo(ctx)
+func (n *nasObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
+	sinfo := n.ObjectLayer.StorageInfo(ctx, false)
 	sinfo.Backend.GatewayOnline = sinfo.Backend.Type == minio.BackendFS
 	sinfo.Backend.Type = minio.BackendGateway
 	return sinfo
@@ -120,6 +120,6 @@ type nasObjects struct {
 
 // IsReady returns whether the layer is ready to take requests.
 func (n *nasObjects) IsReady(ctx context.Context) bool {
-	sinfo := n.ObjectLayer.StorageInfo(ctx)
+	sinfo := n.ObjectLayer.StorageInfo(ctx, false)
 	return sinfo.Backend.Type == minio.BackendFS
 }

cmd/gateway/oss/gateway-oss.go

@@ -320,7 +320,7 @@ func (l *ossObjects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo is not relevant to OSS backend.
-func (l *ossObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (l *ossObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint)
 	return si

cmd/gateway/s3/gateway-s3.go

@@ -272,7 +272,7 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo is not relevant to S3 backend.
-func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String())
 	return si

cmd/healthcheck-handler.go

@@ -60,7 +60,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if !globalIsXL && !globalIsDistXL {
-		s := objLayer.StorageInfo(ctx)
+		s := objLayer.StorageInfo(ctx, false)
 		if s.Backend.Type == BackendGateway {
 			if !s.Backend.GatewayOnline {
 				writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)

cmd/metrics.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"context"
 	"net/http"
-	"strings"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/prometheus/client_golang/prometheus"
@@ -89,48 +88,35 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 		return
 	}
 
-	storageAPIs := []StorageAPI{}
-	for _, ep := range globalEndpoints {
-		for _, endpoint := range ep.Endpoints {
-			if endpoint.IsLocal {
-				// Construct storageAPIs.
-				sAPI, _ := newStorageAPI(endpoint)
-				storageAPIs = append(storageAPIs, sAPI)
-			}
-		}
-	}
+	storageInfo := objLayer.StorageInfo(context.Background(), true)
 
-	disksInfo, onlineDisks, offlineDisks := getDisksInfo(storageAPIs)
+	offlineDisks := storageInfo.Backend.OfflineDisks
+	onlineDisks := storageInfo.Backend.OnlineDisks
 	totalDisks := offlineDisks.Merge(onlineDisks)
 
-	for _, offDisks := range offlineDisks {
-		// MinIO Offline Disks per node
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				prometheus.BuildFQName("minio", "disks", "offline"),
-				"Total number of offline disks in current MinIO server instance",
-				nil, nil),
-			prometheus.GaugeValue,
-			float64(offDisks),
-		)
-	}
+	// MinIO Offline Disks per node
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "disks", "offline"),
+			"Total number of offline disks in current MinIO server instance",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(offlineDisks.Sum()),
+	)
 
-	for _, totDisks := range totalDisks {
-		// MinIO Total Disks per node
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				prometheus.BuildFQName("minio", "disks", "total"),
-				"Total number of disks for current MinIO server instance",
-				nil, nil),
-			prometheus.GaugeValue,
-			float64(totDisks),
-		)
-	}
+	// MinIO Total Disks per node
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "disks", "total"),
+			"Total number of disks for current MinIO server instance",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(totalDisks.Sum()),
+	)
 
-	localPeer := GetLocalPeer(globalEndpoints)
-	for _, di := range disksInfo {
-		// Trim the host
-		absPath := strings.TrimPrefix(di.RelativePath, localPeer)
+	for i := 0; i < len(storageInfo.Total); i++ {
+		mountPath, total, free := storageInfo.MountPaths[i], storageInfo.Total[i],
+			storageInfo.Available[i]
 
 		// Total disk usage by the disk
 		ch <- prometheus.MustNewConstMetric(
@@ -139,8 +125,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 				"Total disk storage used on the disk",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
-			float64(di.Total-di.Free),
-			absPath,
+			float64(total-free),
+			mountPath,
 		)
 
 		// Total available space in the disk
@@ -150,8 +136,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 				"Total available space left on the disk",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
-			float64(di.Free),
-			absPath,
+			float64(free),
+			mountPath,
 		)
 
 		// Total storage space of the disk
@@ -161,8 +147,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 				"Total space on the disk",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
-			float64(di.Total),
-			absPath,
+			float64(total),
+			mountPath,
 		)
 	}
 

cmd/object-api-interface.go

@@ -60,7 +60,7 @@ type ObjectLayer interface {
 	// Storage operations.
 	Shutdown(context.Context) error
 	CrawlAndGetDataUsage(context.Context, <-chan struct{}) DataUsageInfo
-	StorageInfo(context.Context) StorageInfo
+	StorageInfo(ctx context.Context, local bool) StorageInfo // local queries only local disks
 
 	// Bucket operations.
 	MakeBucketWithLocation(ctx context.Context, bucket string, location string) error

cmd/posix.go

@@ -368,11 +368,11 @@ func (s *posix) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, erro
 // DiskInfo is an extended type which returns current
 // disk usage per path.
 type DiskInfo struct {
-	Total        uint64
-	Free         uint64
-	Used         uint64
-	RootDisk     bool
-	RelativePath string
+	Total     uint64
+	Free      uint64
+	Used      uint64
+	RootDisk  bool
+	MountPath string
 }
 
 // DiskInfo provides current information about disk space usage,
@@ -398,17 +398,12 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
 		return info, err
 	}
 
-	localPeer := ""
-	if globalIsDistXL {
-		localPeer = GetLocalPeer(globalEndpoints)
-	}
-
 	return DiskInfo{
-		Total:        di.Total,
-		Free:         di.Free,
-		Used:         used,
-		RootDisk:     rootDisk,
-		RelativePath: localPeer + s.diskPath,
+		Total:     di.Total,
+		Free:      di.Free,
+		Used:      used,
+		RootDisk:  rootDisk,
+		MountPath: s.diskPath,
 	}, nil
 }
 

cmd/server-startup-msg.go

@@ -54,7 +54,7 @@ func printStartupSafeModeMessage(apiEndpoints []string, err error) {
 	// Object layer is initialized then print StorageInfo in safe mode.
 	objAPI := newObjectLayerWithoutSafeModeFn()
 	if objAPI != nil {
-		if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background())); msg != "" {
+		if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background(), false)); msg != "" {
 			logStartupMessage(msg)
 		}
 	}
@@ -116,7 +116,7 @@ func printStartupMessage(apiEndpoints []string) {
 	// Object layer is initialized then print StorageInfo.
 	objAPI := newObjectLayerFn()
 	if objAPI != nil {
-		printStorageInfo(objAPI.StorageInfo(context.Background()))
+		printStorageInfo(objAPI.StorageInfo(context.Background(), false))
 	}
 
 	// Prints credential, region and browser access.

cmd/web-handlers.go

@@ -129,7 +129,7 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, re
 	if authErr != nil {
 		return toJSONError(ctx, authErr)
 	}
-	reply.StorageInfo = objectAPI.StorageInfo(ctx)
+	reply.StorageInfo = objectAPI.StorageInfo(ctx, false)
 	reply.UIVersion = browser.UIVersion
 	return nil
 }

cmd/xl-sets.go

@@ -340,7 +340,7 @@ func (s *xlSets) NewNSLock(ctx context.Context, bucket string, object string) RW
 }
 
 // StorageInfo - combines output of StorageInfo across all erasure coded object sets.
-func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
+func (s *xlSets) StorageInfo(ctx context.Context, local bool) StorageInfo {
 	var storageInfo StorageInfo
 
 	storageInfos := make([]StorageInfo, len(s.sets))
@@ -350,7 +350,7 @@ func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
 	for index := range s.sets {
 		index := index
 		g.Go(func() error {
-			storageInfos[index] = s.sets[index].StorageInfo(ctx)
+			storageInfos[index] = s.sets[index].StorageInfo(ctx, local)
 			return nil
 		}, index)
 	}

cmd/xl-v1.go

@@ -19,14 +19,12 @@ package cmd
 import (
 	"context"
 	"sort"
-	"strings"
 	"sync"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
 	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/madmin"
-	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
@@ -115,34 +113,27 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin
 		}, index)
 	}
 
-	getPeerAddress := func(diskPath string) (string, error) {
-		hostPort := strings.Split(diskPath, SlashSeparator)[0]
-		// Host will be empty for xl/fs disk paths.
-		if hostPort == "" {
-			return "", nil
-		}
-		thisAddr, err := xnet.ParseHost(hostPort)
-		if err != nil {
-			return "", err
-		}
-		return thisAddr.String(), nil
-	}
-
 	onlineDisks = make(madmin.BackendDisks)
 	offlineDisks = make(madmin.BackendDisks)
+
+	localNodeAddr := GetLocalPeer(globalEndpoints)
+
 	// Wait for the routines.
-	for i, err := range g.Wait() {
-		peerAddr, pErr := getPeerAddress(disksInfo[i].RelativePath)
-		if pErr != nil {
+	for i, diskInfoErr := range g.Wait() {
+		if disks[i] == nil {
 			continue
 		}
+		peerAddr := disks[i].Hostname()
+		if peerAddr == "" {
+			peerAddr = localNodeAddr
+		}
 		if _, ok := offlineDisks[peerAddr]; !ok {
 			offlineDisks[peerAddr] = 0
 		}
 		if _, ok := onlineDisks[peerAddr]; !ok {
 			onlineDisks[peerAddr] = 0
 		}
-		if err != nil {
+		if diskInfoErr != nil {
 			offlineDisks[peerAddr]++
 			continue
 		}
@@ -170,7 +161,7 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
 		usedList[i] = di.Used
 		totalList[i] = di.Total
 		availableList[i] = di.Free
-		mountPaths[i] = di.RelativePath
+		mountPaths[i] = di.MountPath
 	}
 
 	storageInfo := StorageInfo{
@@ -188,8 +179,19 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
 }
 
 // StorageInfo - returns underlying storage statistics.
-func (xl xlObjects) StorageInfo(ctx context.Context) StorageInfo {
-	return getStorageInfo(xl.getDisks())
+func (xl xlObjects) StorageInfo(ctx context.Context, local bool) StorageInfo {
+	var disks []StorageAPI
+	if !local {
+		disks = xl.getDisks()
+	} else {
+		for _, d := range xl.getDisks() {
+			if d.Hostname() == "" {
+				// Append this local disk since local flag is true
+				disks = append(disks, d)
+			}
+		}
+	}
+	return getStorageInfo(disks)
 }
 
 // GetMetrics - is not implemented and shouldn't be called.

cmd/xl-zones.go

@@ -131,7 +131,7 @@ func (z *xlZones) getZonesAvailableSpace(ctx context.Context) zonesAvailableSpac
 	for index := range z.zones {
 		index := index
 		g.Go(func() error {
-			storageInfos[index] = z.zones[index].StorageInfo(ctx)
+			storageInfos[index] = z.zones[index].StorageInfo(ctx, false)
 			return nil
 		}, index)
 	}
@@ -176,9 +176,9 @@ func (z *xlZones) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo {
+func (z *xlZones) StorageInfo(ctx context.Context, local bool) StorageInfo {
 	if z.SingleZone() {
-		return z.zones[0].StorageInfo(ctx)
+		return z.zones[0].StorageInfo(ctx, local)
 	}
 
 	var storageInfo StorageInfo
@@ -188,7 +188,7 @@ func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo {
 	for index := range z.zones {
 		index := index
 		g.Go(func() error {
-			storageInfos[index] = z.zones[index].StorageInfo(ctx)
+			storageInfos[index] = z.zones[index].StorageInfo(ctx, local)
 			return nil
 		}, index)
 	}