metrics: Use StorageInfo() instead to have consistent info (#9006)

Metrics used to have its own code to calculate offline disks.
StorageInfo() was avoided because it is an expensive operation
by sending calls to all nodes.

To make metrics & server info share the same code, a new
argument `local` is added to StorageInfo() so it will only
query local disks when needed.

Metrics now calls StorageInfo() as server info handler does
but with the local flag set to false.

Co-authored-by: Praveen raj Mani <praveen@minio.io>
Co-authored-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Anis Elleuch 2020-02-20 04:51:33 +01:00 committed by GitHub
parent 02a59a04d1
commit d4dcf1d722
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 89 additions and 106 deletions

View File

@ -276,7 +276,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
return
}
storageInfo := objectAPI.StorageInfo(ctx)
storageInfo := objectAPI.StorageInfo(ctx, false)
// Marshal API response
jsonBytes, err := json.Marshal(storageInfo)
@ -897,7 +897,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
}
// find number of disks in the setup
info := objectAPI.StorageInfo(ctx)
info := objectAPI.StorageInfo(ctx, false)
numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
healPath := pathJoin(hip.bucket, hip.objPrefix)
@ -1410,7 +1410,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
notifyTarget := fetchLambdaInfo(cfg)
// Fetching the Storage information
storageInfo := objectAPI.StorageInfo(ctx)
storageInfo := objectAPI.StorageInfo(ctx, false)
var OnDisks int
var OffDisks int

View File

@ -127,7 +127,7 @@ func startBackgroundHealing() {
// Launch the background healer sequence to track
// background healing operations
info := objAPI.StorageInfo(ctx)
info := objAPI.StorageInfo(ctx, false)
numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
nh := newBgHealSequence(numDisks)
globalBackgroundHealState.LaunchNewHealSequence(nh)

View File

@ -202,7 +202,7 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
}
// StorageInfo - returns underlying storage statistics.
func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo {
atomic.AddInt64(&fs.activeIOCount, 1)
defer func() {

View File

@ -481,7 +481,7 @@ func (a *azureObjects) Shutdown(ctx context.Context) error {
}
// StorageInfo - Not relevant to Azure backend.
func (a *azureObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, a.httpClient, a.endpoint)
return si

View File

@ -218,7 +218,7 @@ func (l *b2Objects) Shutdown(ctx context.Context) error {
}
// StorageInfo is not relevant to B2 backend.
func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *b2Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://api.backblazeb2.com/b2api/v1")
return si

View File

@ -412,7 +412,7 @@ func (l *gcsGateway) Shutdown(ctx context.Context) error {
}
// StorageInfo - Not relevant to GCS backend.
func (l *gcsGateway) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com")
return si

View File

@ -203,7 +203,7 @@ func (n *hdfsObjects) Shutdown(ctx context.Context) error {
return n.clnt.Close()
}
func (n *hdfsObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
func (n *hdfsObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
fsInfo, err := n.clnt.StatFs()
if err != nil {
return minio.StorageInfo{}

View File

@ -106,8 +106,8 @@ func (n *nasObjects) IsListenBucketSupported() bool {
return false
}
func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
sinfo := n.ObjectLayer.StorageInfo(ctx)
func (n *nasObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
sinfo := n.ObjectLayer.StorageInfo(ctx, false)
sinfo.Backend.GatewayOnline = sinfo.Backend.Type == minio.BackendFS
sinfo.Backend.Type = minio.BackendGateway
return sinfo
@ -120,6 +120,6 @@ type nasObjects struct {
// IsReady returns whether the layer is ready to take requests.
func (n *nasObjects) IsReady(ctx context.Context) bool {
sinfo := n.ObjectLayer.StorageInfo(ctx)
sinfo := n.ObjectLayer.StorageInfo(ctx, false)
return sinfo.Backend.Type == minio.BackendFS
}

View File

@ -320,7 +320,7 @@ func (l *ossObjects) Shutdown(ctx context.Context) error {
}
// StorageInfo is not relevant to OSS backend.
func (l *ossObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *ossObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint)
return si

View File

@ -272,7 +272,7 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
}
// StorageInfo is not relevant to S3 backend.
func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String())
return si

View File

@ -60,7 +60,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
}
if !globalIsXL && !globalIsDistXL {
s := objLayer.StorageInfo(ctx)
s := objLayer.StorageInfo(ctx, false)
if s.Backend.Type == BackendGateway {
if !s.Backend.GatewayOnline {
writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)

View File

@ -19,7 +19,6 @@ package cmd
import (
"context"
"net/http"
"strings"
"github.com/minio/minio/cmd/logger"
"github.com/prometheus/client_golang/prometheus"
@ -89,21 +88,12 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
return
}
storageAPIs := []StorageAPI{}
for _, ep := range globalEndpoints {
for _, endpoint := range ep.Endpoints {
if endpoint.IsLocal {
// Construct storageAPIs.
sAPI, _ := newStorageAPI(endpoint)
storageAPIs = append(storageAPIs, sAPI)
}
}
}
storageInfo := objLayer.StorageInfo(context.Background(), true)
disksInfo, onlineDisks, offlineDisks := getDisksInfo(storageAPIs)
offlineDisks := storageInfo.Backend.OfflineDisks
onlineDisks := storageInfo.Backend.OnlineDisks
totalDisks := offlineDisks.Merge(onlineDisks)
for _, offDisks := range offlineDisks {
// MinIO Offline Disks per node
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
@ -111,11 +101,9 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total number of offline disks in current MinIO server instance",
nil, nil),
prometheus.GaugeValue,
float64(offDisks),
float64(offlineDisks.Sum()),
)
}
for _, totDisks := range totalDisks {
// MinIO Total Disks per node
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
@ -123,14 +111,12 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total number of disks for current MinIO server instance",
nil, nil),
prometheus.GaugeValue,
float64(totDisks),
float64(totalDisks.Sum()),
)
}
localPeer := GetLocalPeer(globalEndpoints)
for _, di := range disksInfo {
// Trim the host
absPath := strings.TrimPrefix(di.RelativePath, localPeer)
for i := 0; i < len(storageInfo.Total); i++ {
mountPath, total, free := storageInfo.MountPaths[i], storageInfo.Total[i],
storageInfo.Available[i]
// Total disk usage by the disk
ch <- prometheus.MustNewConstMetric(
@ -139,8 +125,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total disk storage used on the disk",
[]string{"disk"}, nil),
prometheus.GaugeValue,
float64(di.Total-di.Free),
absPath,
float64(total-free),
mountPath,
)
// Total available space in the disk
@ -150,8 +136,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total available space left on the disk",
[]string{"disk"}, nil),
prometheus.GaugeValue,
float64(di.Free),
absPath,
float64(free),
mountPath,
)
// Total storage space of the disk
@ -161,8 +147,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total space on the disk",
[]string{"disk"}, nil),
prometheus.GaugeValue,
float64(di.Total),
absPath,
float64(total),
mountPath,
)
}

View File

@ -60,7 +60,7 @@ type ObjectLayer interface {
// Storage operations.
Shutdown(context.Context) error
CrawlAndGetDataUsage(context.Context, <-chan struct{}) DataUsageInfo
StorageInfo(context.Context) StorageInfo
StorageInfo(ctx context.Context, local bool) StorageInfo // local queries only local disks
// Bucket operations.
MakeBucketWithLocation(ctx context.Context, bucket string, location string) error

View File

@ -372,7 +372,7 @@ type DiskInfo struct {
Free uint64
Used uint64
RootDisk bool
RelativePath string
MountPath string
}
// DiskInfo provides current information about disk space usage,
@ -398,17 +398,12 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
return info, err
}
localPeer := ""
if globalIsDistXL {
localPeer = GetLocalPeer(globalEndpoints)
}
return DiskInfo{
Total: di.Total,
Free: di.Free,
Used: used,
RootDisk: rootDisk,
RelativePath: localPeer + s.diskPath,
MountPath: s.diskPath,
}, nil
}

View File

@ -54,7 +54,7 @@ func printStartupSafeModeMessage(apiEndpoints []string, err error) {
// Object layer is initialized then print StorageInfo in safe mode.
objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI != nil {
if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background())); msg != "" {
if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background(), false)); msg != "" {
logStartupMessage(msg)
}
}
@ -116,7 +116,7 @@ func printStartupMessage(apiEndpoints []string) {
// Object layer is initialized then print StorageInfo.
objAPI := newObjectLayerFn()
if objAPI != nil {
printStorageInfo(objAPI.StorageInfo(context.Background()))
printStorageInfo(objAPI.StorageInfo(context.Background(), false))
}
// Prints credential, region and browser access.

View File

@ -129,7 +129,7 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, re
if authErr != nil {
return toJSONError(ctx, authErr)
}
reply.StorageInfo = objectAPI.StorageInfo(ctx)
reply.StorageInfo = objectAPI.StorageInfo(ctx, false)
reply.UIVersion = browser.UIVersion
return nil
}

View File

@ -340,7 +340,7 @@ func (s *xlSets) NewNSLock(ctx context.Context, bucket string, object string) RW
}
// StorageInfo - combines output of StorageInfo across all erasure coded object sets.
func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
func (s *xlSets) StorageInfo(ctx context.Context, local bool) StorageInfo {
var storageInfo StorageInfo
storageInfos := make([]StorageInfo, len(s.sets))
@ -350,7 +350,7 @@ func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
for index := range s.sets {
index := index
g.Go(func() error {
storageInfos[index] = s.sets[index].StorageInfo(ctx)
storageInfos[index] = s.sets[index].StorageInfo(ctx, local)
return nil
}, index)
}

View File

@ -19,14 +19,12 @@ package cmd
import (
"context"
"sort"
"strings"
"sync"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/bpool"
"github.com/minio/minio/pkg/dsync"
"github.com/minio/minio/pkg/madmin"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/sync/errgroup"
)
@ -115,34 +113,27 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin
}, index)
}
getPeerAddress := func(diskPath string) (string, error) {
hostPort := strings.Split(diskPath, SlashSeparator)[0]
// Host will be empty for xl/fs disk paths.
if hostPort == "" {
return "", nil
}
thisAddr, err := xnet.ParseHost(hostPort)
if err != nil {
return "", err
}
return thisAddr.String(), nil
}
onlineDisks = make(madmin.BackendDisks)
offlineDisks = make(madmin.BackendDisks)
localNodeAddr := GetLocalPeer(globalEndpoints)
// Wait for the routines.
for i, err := range g.Wait() {
peerAddr, pErr := getPeerAddress(disksInfo[i].RelativePath)
if pErr != nil {
for i, diskInfoErr := range g.Wait() {
if disks[i] == nil {
continue
}
peerAddr := disks[i].Hostname()
if peerAddr == "" {
peerAddr = localNodeAddr
}
if _, ok := offlineDisks[peerAddr]; !ok {
offlineDisks[peerAddr] = 0
}
if _, ok := onlineDisks[peerAddr]; !ok {
onlineDisks[peerAddr] = 0
}
if err != nil {
if diskInfoErr != nil {
offlineDisks[peerAddr]++
continue
}
@ -170,7 +161,7 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
usedList[i] = di.Used
totalList[i] = di.Total
availableList[i] = di.Free
mountPaths[i] = di.RelativePath
mountPaths[i] = di.MountPath
}
storageInfo := StorageInfo{
@ -188,8 +179,19 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
}
// StorageInfo - returns underlying storage statistics.
func (xl xlObjects) StorageInfo(ctx context.Context) StorageInfo {
return getStorageInfo(xl.getDisks())
func (xl xlObjects) StorageInfo(ctx context.Context, local bool) StorageInfo {
var disks []StorageAPI
if !local {
disks = xl.getDisks()
} else {
for _, d := range xl.getDisks() {
if d.Hostname() == "" {
// Append this local disk since local flag is true
disks = append(disks, d)
}
}
}
return getStorageInfo(disks)
}
// GetMetrics - is not implemented and shouldn't be called.

View File

@ -131,7 +131,7 @@ func (z *xlZones) getZonesAvailableSpace(ctx context.Context) zonesAvailableSpac
for index := range z.zones {
index := index
g.Go(func() error {
storageInfos[index] = z.zones[index].StorageInfo(ctx)
storageInfos[index] = z.zones[index].StorageInfo(ctx, false)
return nil
}, index)
}
@ -176,9 +176,9 @@ func (z *xlZones) Shutdown(ctx context.Context) error {
return nil
}
func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo {
func (z *xlZones) StorageInfo(ctx context.Context, local bool) StorageInfo {
if z.SingleZone() {
return z.zones[0].StorageInfo(ctx)
return z.zones[0].StorageInfo(ctx, local)
}
var storageInfo StorageInfo
@ -188,7 +188,7 @@ func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo {
for index := range z.zones {
index := index
g.Go(func() error {
storageInfos[index] = z.zones[index].StorageInfo(ctx)
storageInfos[index] = z.zones[index].StorageInfo(ctx, local)
return nil
}, index)
}