metrics: Use StorageInfo() instead to have consistent info (#9006)

Metrics used to have its own code to calculate offline disks.
StorageInfo() was avoided because it is an expensive operation
by sending calls to all nodes.

To make metrics & server info share the same code, a new
argument `local` is added to StorageInfo() so it will only
query local disks when needed.

Metrics now calls StorageInfo() as server info handler does
but with the local flag set to false.

Co-authored-by: Praveen raj Mani <praveen@minio.io>
Co-authored-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Anis Elleuch 2020-02-20 04:51:33 +01:00 committed by GitHub
parent 02a59a04d1
commit d4dcf1d722
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 89 additions and 106 deletions

View File

@ -276,7 +276,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
return return
} }
storageInfo := objectAPI.StorageInfo(ctx) storageInfo := objectAPI.StorageInfo(ctx, false)
// Marshal API response // Marshal API response
jsonBytes, err := json.Marshal(storageInfo) jsonBytes, err := json.Marshal(storageInfo)
@ -897,7 +897,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
} }
// find number of disks in the setup // find number of disks in the setup
info := objectAPI.StorageInfo(ctx) info := objectAPI.StorageInfo(ctx, false)
numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum() numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
healPath := pathJoin(hip.bucket, hip.objPrefix) healPath := pathJoin(hip.bucket, hip.objPrefix)
@ -1410,7 +1410,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
notifyTarget := fetchLambdaInfo(cfg) notifyTarget := fetchLambdaInfo(cfg)
// Fetching the Storage information // Fetching the Storage information
storageInfo := objectAPI.StorageInfo(ctx) storageInfo := objectAPI.StorageInfo(ctx, false)
var OnDisks int var OnDisks int
var OffDisks int var OffDisks int

View File

@ -127,7 +127,7 @@ func startBackgroundHealing() {
// Launch the background healer sequence to track // Launch the background healer sequence to track
// background healing operations // background healing operations
info := objAPI.StorageInfo(ctx) info := objAPI.StorageInfo(ctx, false)
numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum() numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
nh := newBgHealSequence(numDisks) nh := newBgHealSequence(numDisks)
globalBackgroundHealState.LaunchNewHealSequence(nh) globalBackgroundHealState.LaunchNewHealSequence(nh)

View File

@ -202,7 +202,7 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
} }
// StorageInfo - returns underlying storage statistics. // StorageInfo - returns underlying storage statistics.
func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo { func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo {
atomic.AddInt64(&fs.activeIOCount, 1) atomic.AddInt64(&fs.activeIOCount, 1)
defer func() { defer func() {

View File

@ -481,7 +481,7 @@ func (a *azureObjects) Shutdown(ctx context.Context) error {
} }
// StorageInfo - Not relevant to Azure backend. // StorageInfo - Not relevant to Azure backend.
func (a *azureObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, a.httpClient, a.endpoint) si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, a.httpClient, a.endpoint)
return si return si

View File

@ -218,7 +218,7 @@ func (l *b2Objects) Shutdown(ctx context.Context) error {
} }
// StorageInfo is not relevant to B2 backend. // StorageInfo is not relevant to B2 backend.
func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { func (l *b2Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://api.backblazeb2.com/b2api/v1") si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://api.backblazeb2.com/b2api/v1")
return si return si

View File

@ -412,7 +412,7 @@ func (l *gcsGateway) Shutdown(ctx context.Context) error {
} }
// StorageInfo - Not relevant to GCS backend. // StorageInfo - Not relevant to GCS backend.
func (l *gcsGateway) StorageInfo(ctx context.Context) (si minio.StorageInfo) { func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com") si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com")
return si return si

View File

@ -203,7 +203,7 @@ func (n *hdfsObjects) Shutdown(ctx context.Context) error {
return n.clnt.Close() return n.clnt.Close()
} }
func (n *hdfsObjects) StorageInfo(ctx context.Context) minio.StorageInfo { func (n *hdfsObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
fsInfo, err := n.clnt.StatFs() fsInfo, err := n.clnt.StatFs()
if err != nil { if err != nil {
return minio.StorageInfo{} return minio.StorageInfo{}

View File

@ -106,8 +106,8 @@ func (n *nasObjects) IsListenBucketSupported() bool {
return false return false
} }
func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo { func (n *nasObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
sinfo := n.ObjectLayer.StorageInfo(ctx) sinfo := n.ObjectLayer.StorageInfo(ctx, false)
sinfo.Backend.GatewayOnline = sinfo.Backend.Type == minio.BackendFS sinfo.Backend.GatewayOnline = sinfo.Backend.Type == minio.BackendFS
sinfo.Backend.Type = minio.BackendGateway sinfo.Backend.Type = minio.BackendGateway
return sinfo return sinfo
@ -120,6 +120,6 @@ type nasObjects struct {
// IsReady returns whether the layer is ready to take requests. // IsReady returns whether the layer is ready to take requests.
func (n *nasObjects) IsReady(ctx context.Context) bool { func (n *nasObjects) IsReady(ctx context.Context) bool {
sinfo := n.ObjectLayer.StorageInfo(ctx) sinfo := n.ObjectLayer.StorageInfo(ctx, false)
return sinfo.Backend.Type == minio.BackendFS return sinfo.Backend.Type == minio.BackendFS
} }

View File

@ -320,7 +320,7 @@ func (l *ossObjects) Shutdown(ctx context.Context) error {
} }
// StorageInfo is not relevant to OSS backend. // StorageInfo is not relevant to OSS backend.
func (l *ossObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { func (l *ossObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint) si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint)
return si return si

View File

@ -272,7 +272,7 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
} }
// StorageInfo is not relevant to S3 backend. // StorageInfo is not relevant to S3 backend.
func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String()) si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String())
return si return si

View File

@ -60,7 +60,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
} }
if !globalIsXL && !globalIsDistXL { if !globalIsXL && !globalIsDistXL {
s := objLayer.StorageInfo(ctx) s := objLayer.StorageInfo(ctx, false)
if s.Backend.Type == BackendGateway { if s.Backend.Type == BackendGateway {
if !s.Backend.GatewayOnline { if !s.Backend.GatewayOnline {
writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)

View File

@ -19,7 +19,6 @@ package cmd
import ( import (
"context" "context"
"net/http" "net/http"
"strings"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
@ -89,21 +88,12 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
return return
} }
storageAPIs := []StorageAPI{} storageInfo := objLayer.StorageInfo(context.Background(), true)
for _, ep := range globalEndpoints {
for _, endpoint := range ep.Endpoints {
if endpoint.IsLocal {
// Construct storageAPIs.
sAPI, _ := newStorageAPI(endpoint)
storageAPIs = append(storageAPIs, sAPI)
}
}
}
disksInfo, onlineDisks, offlineDisks := getDisksInfo(storageAPIs) offlineDisks := storageInfo.Backend.OfflineDisks
onlineDisks := storageInfo.Backend.OnlineDisks
totalDisks := offlineDisks.Merge(onlineDisks) totalDisks := offlineDisks.Merge(onlineDisks)
for _, offDisks := range offlineDisks {
// MinIO Offline Disks per node // MinIO Offline Disks per node
ch <- prometheus.MustNewConstMetric( ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc( prometheus.NewDesc(
@ -111,11 +101,9 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total number of offline disks in current MinIO server instance", "Total number of offline disks in current MinIO server instance",
nil, nil), nil, nil),
prometheus.GaugeValue, prometheus.GaugeValue,
float64(offDisks), float64(offlineDisks.Sum()),
) )
}
for _, totDisks := range totalDisks {
// MinIO Total Disks per node // MinIO Total Disks per node
ch <- prometheus.MustNewConstMetric( ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc( prometheus.NewDesc(
@ -123,14 +111,12 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total number of disks for current MinIO server instance", "Total number of disks for current MinIO server instance",
nil, nil), nil, nil),
prometheus.GaugeValue, prometheus.GaugeValue,
float64(totDisks), float64(totalDisks.Sum()),
) )
}
localPeer := GetLocalPeer(globalEndpoints) for i := 0; i < len(storageInfo.Total); i++ {
for _, di := range disksInfo { mountPath, total, free := storageInfo.MountPaths[i], storageInfo.Total[i],
// Trim the host storageInfo.Available[i]
absPath := strings.TrimPrefix(di.RelativePath, localPeer)
// Total disk usage by the disk // Total disk usage by the disk
ch <- prometheus.MustNewConstMetric( ch <- prometheus.MustNewConstMetric(
@ -139,8 +125,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total disk storage used on the disk", "Total disk storage used on the disk",
[]string{"disk"}, nil), []string{"disk"}, nil),
prometheus.GaugeValue, prometheus.GaugeValue,
float64(di.Total-di.Free), float64(total-free),
absPath, mountPath,
) )
// Total available space in the disk // Total available space in the disk
@ -150,8 +136,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total available space left on the disk", "Total available space left on the disk",
[]string{"disk"}, nil), []string{"disk"}, nil),
prometheus.GaugeValue, prometheus.GaugeValue,
float64(di.Free), float64(free),
absPath, mountPath,
) )
// Total storage space of the disk // Total storage space of the disk
@ -161,8 +147,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total space on the disk", "Total space on the disk",
[]string{"disk"}, nil), []string{"disk"}, nil),
prometheus.GaugeValue, prometheus.GaugeValue,
float64(di.Total), float64(total),
absPath, mountPath,
) )
} }

View File

@ -60,7 +60,7 @@ type ObjectLayer interface {
// Storage operations. // Storage operations.
Shutdown(context.Context) error Shutdown(context.Context) error
CrawlAndGetDataUsage(context.Context, <-chan struct{}) DataUsageInfo CrawlAndGetDataUsage(context.Context, <-chan struct{}) DataUsageInfo
StorageInfo(context.Context) StorageInfo StorageInfo(ctx context.Context, local bool) StorageInfo // local queries only local disks
// Bucket operations. // Bucket operations.
MakeBucketWithLocation(ctx context.Context, bucket string, location string) error MakeBucketWithLocation(ctx context.Context, bucket string, location string) error

View File

@ -372,7 +372,7 @@ type DiskInfo struct {
Free uint64 Free uint64
Used uint64 Used uint64
RootDisk bool RootDisk bool
RelativePath string MountPath string
} }
// DiskInfo provides current information about disk space usage, // DiskInfo provides current information about disk space usage,
@ -398,17 +398,12 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
return info, err return info, err
} }
localPeer := ""
if globalIsDistXL {
localPeer = GetLocalPeer(globalEndpoints)
}
return DiskInfo{ return DiskInfo{
Total: di.Total, Total: di.Total,
Free: di.Free, Free: di.Free,
Used: used, Used: used,
RootDisk: rootDisk, RootDisk: rootDisk,
RelativePath: localPeer + s.diskPath, MountPath: s.diskPath,
}, nil }, nil
} }

View File

@ -54,7 +54,7 @@ func printStartupSafeModeMessage(apiEndpoints []string, err error) {
// Object layer is initialized then print StorageInfo in safe mode. // Object layer is initialized then print StorageInfo in safe mode.
objAPI := newObjectLayerWithoutSafeModeFn() objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI != nil { if objAPI != nil {
if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background())); msg != "" { if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background(), false)); msg != "" {
logStartupMessage(msg) logStartupMessage(msg)
} }
} }
@ -116,7 +116,7 @@ func printStartupMessage(apiEndpoints []string) {
// Object layer is initialized then print StorageInfo. // Object layer is initialized then print StorageInfo.
objAPI := newObjectLayerFn() objAPI := newObjectLayerFn()
if objAPI != nil { if objAPI != nil {
printStorageInfo(objAPI.StorageInfo(context.Background())) printStorageInfo(objAPI.StorageInfo(context.Background(), false))
} }
// Prints credential, region and browser access. // Prints credential, region and browser access.

View File

@ -129,7 +129,7 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, re
if authErr != nil { if authErr != nil {
return toJSONError(ctx, authErr) return toJSONError(ctx, authErr)
} }
reply.StorageInfo = objectAPI.StorageInfo(ctx) reply.StorageInfo = objectAPI.StorageInfo(ctx, false)
reply.UIVersion = browser.UIVersion reply.UIVersion = browser.UIVersion
return nil return nil
} }

View File

@ -340,7 +340,7 @@ func (s *xlSets) NewNSLock(ctx context.Context, bucket string, object string) RW
} }
// StorageInfo - combines output of StorageInfo across all erasure coded object sets. // StorageInfo - combines output of StorageInfo across all erasure coded object sets.
func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo { func (s *xlSets) StorageInfo(ctx context.Context, local bool) StorageInfo {
var storageInfo StorageInfo var storageInfo StorageInfo
storageInfos := make([]StorageInfo, len(s.sets)) storageInfos := make([]StorageInfo, len(s.sets))
@ -350,7 +350,7 @@ func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
for index := range s.sets { for index := range s.sets {
index := index index := index
g.Go(func() error { g.Go(func() error {
storageInfos[index] = s.sets[index].StorageInfo(ctx) storageInfos[index] = s.sets[index].StorageInfo(ctx, local)
return nil return nil
}, index) }, index)
} }

View File

@ -19,14 +19,12 @@ package cmd
import ( import (
"context" "context"
"sort" "sort"
"strings"
"sync" "sync"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/bpool" "github.com/minio/minio/pkg/bpool"
"github.com/minio/minio/pkg/dsync" "github.com/minio/minio/pkg/dsync"
"github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/madmin"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/sync/errgroup" "github.com/minio/minio/pkg/sync/errgroup"
) )
@ -115,34 +113,27 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin
}, index) }, index)
} }
getPeerAddress := func(diskPath string) (string, error) {
hostPort := strings.Split(diskPath, SlashSeparator)[0]
// Host will be empty for xl/fs disk paths.
if hostPort == "" {
return "", nil
}
thisAddr, err := xnet.ParseHost(hostPort)
if err != nil {
return "", err
}
return thisAddr.String(), nil
}
onlineDisks = make(madmin.BackendDisks) onlineDisks = make(madmin.BackendDisks)
offlineDisks = make(madmin.BackendDisks) offlineDisks = make(madmin.BackendDisks)
localNodeAddr := GetLocalPeer(globalEndpoints)
// Wait for the routines. // Wait for the routines.
for i, err := range g.Wait() { for i, diskInfoErr := range g.Wait() {
peerAddr, pErr := getPeerAddress(disksInfo[i].RelativePath) if disks[i] == nil {
if pErr != nil {
continue continue
} }
peerAddr := disks[i].Hostname()
if peerAddr == "" {
peerAddr = localNodeAddr
}
if _, ok := offlineDisks[peerAddr]; !ok { if _, ok := offlineDisks[peerAddr]; !ok {
offlineDisks[peerAddr] = 0 offlineDisks[peerAddr] = 0
} }
if _, ok := onlineDisks[peerAddr]; !ok { if _, ok := onlineDisks[peerAddr]; !ok {
onlineDisks[peerAddr] = 0 onlineDisks[peerAddr] = 0
} }
if err != nil { if diskInfoErr != nil {
offlineDisks[peerAddr]++ offlineDisks[peerAddr]++
continue continue
} }
@ -170,7 +161,7 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
usedList[i] = di.Used usedList[i] = di.Used
totalList[i] = di.Total totalList[i] = di.Total
availableList[i] = di.Free availableList[i] = di.Free
mountPaths[i] = di.RelativePath mountPaths[i] = di.MountPath
} }
storageInfo := StorageInfo{ storageInfo := StorageInfo{
@ -188,8 +179,19 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
} }
// StorageInfo - returns underlying storage statistics. // StorageInfo - returns underlying storage statistics.
func (xl xlObjects) StorageInfo(ctx context.Context) StorageInfo { func (xl xlObjects) StorageInfo(ctx context.Context, local bool) StorageInfo {
return getStorageInfo(xl.getDisks()) var disks []StorageAPI
if !local {
disks = xl.getDisks()
} else {
for _, d := range xl.getDisks() {
if d.Hostname() == "" {
// Append this local disk since local flag is true
disks = append(disks, d)
}
}
}
return getStorageInfo(disks)
} }
// GetMetrics - is not implemented and shouldn't be called. // GetMetrics - is not implemented and shouldn't be called.

View File

@ -131,7 +131,7 @@ func (z *xlZones) getZonesAvailableSpace(ctx context.Context) zonesAvailableSpac
for index := range z.zones { for index := range z.zones {
index := index index := index
g.Go(func() error { g.Go(func() error {
storageInfos[index] = z.zones[index].StorageInfo(ctx) storageInfos[index] = z.zones[index].StorageInfo(ctx, false)
return nil return nil
}, index) }, index)
} }
@ -176,9 +176,9 @@ func (z *xlZones) Shutdown(ctx context.Context) error {
return nil return nil
} }
func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo { func (z *xlZones) StorageInfo(ctx context.Context, local bool) StorageInfo {
if z.SingleZone() { if z.SingleZone() {
return z.zones[0].StorageInfo(ctx) return z.zones[0].StorageInfo(ctx, local)
} }
var storageInfo StorageInfo var storageInfo StorageInfo
@ -188,7 +188,7 @@ func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo {
for index := range z.zones { for index := range z.zones {
index := index index := index
g.Go(func() error { g.Go(func() error {
storageInfos[index] = z.zones[index].StorageInfo(ctx) storageInfos[index] = z.zones[index].StorageInfo(ctx, local)
return nil return nil
}, index) }, index)
} }