heal: Add MRF metrics to background heal API response (#12398)

This commit gathers MRF metrics from 
all nodes in a cluster and return it to the caller. This will show information about the 
number of objects in the MRF queues 
waiting to be healed.
This commit is contained in:
Anis Elleuch
2021-07-16 06:32:06 +01:00
committed by GitHub
parent ead8778305
commit b0b4696a64
12 changed files with 458 additions and 209 deletions

View File

@@ -68,6 +68,17 @@ func newBgHealSequence() *healSequence {
}
}
func getCurrentMRFStatus() madmin.MRFStatus {
mrfInfo := globalMRFState.getCurrentMRFRoundInfo()
return madmin.MRFStatus{
BytesHealed: mrfInfo.bytesHealed,
ItemsHealed: mrfInfo.itemsHealed,
TotalItems: mrfInfo.itemsHealed + mrfInfo.pendingItems,
TotalBytes: mrfInfo.bytesHealed + mrfInfo.pendingBytes,
Started: mrfInfo.triggeredAt,
}
}
// getBackgroundHealStatus will return the
func getBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.BgHealState, bool) {
if globalBackgroundHealState == nil {
@@ -79,13 +90,20 @@ func getBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.BgHealS
return madmin.BgHealState{}, false
}
status := madmin.BgHealState{
ScannedItemsCount: bgSeq.getScannedItemsCount(),
}
if globalMRFState != nil {
status.MRF = map[string]madmin.MRFStatus{
globalLocalNodeName: getCurrentMRFStatus(),
}
}
var healDisksMap = map[string]struct{}{}
for _, ep := range getLocalDisksToHeal() {
healDisksMap[ep.String()] = struct{}{}
}
status := madmin.BgHealState{
ScannedItemsCount: bgSeq.getScannedItemsCount(),
}
if o == nil {
healing := globalBackgroundHealState.getLocalHealingDisks()
@@ -225,12 +243,12 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketIn
ScanMode: madmin.HealNormalScan, Remove: healDeleteDangling}); err != nil {
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
// If not deleted, assume they failed.
tracker.ObjectsFailed++
tracker.ItemsFailed++
tracker.BytesFailed += uint64(version.Size)
logger.LogIf(ctx, err)
}
} else {
tracker.ObjectsHealed++
tracker.ItemsHealed++
tracker.BytesDone += uint64(version.Size)
}
bgSeq.logHeal(madmin.HealItemObject)