heal: Add info about the next background healing round (#9122)

- avoid setting last heal activity when starting self-healing

This can be confusing to users thinking that the self healing
cycle was already performed.

- add info about the next background healing round

cmd/admin-handlers.go

@@ -982,13 +982,24 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{}
+	var aggregatedHealStateResult = madmin.BgHealState{
-	for _, state := range bgHealStates {
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
-		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
-		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+		NextHealRound:     bgHealStates[0].NextHealRound,
-			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
 	}
 
+	bgHealStates = bgHealStates[1:]
+
+	for _, state := range bgHealStates {
+		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
+		}
 	}
 
 	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {

cmd/admin-heal-ops.go

@@ -575,8 +575,6 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	h.lastHealActivity = UTCNow()
-
 	bucketsOnly := true // heal buckets only, not objects.
 	if err := h.healItems(bucketsOnly); err != nil {
 		logger.LogIf(h.ctx, err)

cmd/global-heal.go

@@ -29,7 +29,6 @@ import (
 const (
 	bgHealingUUID = "0000-0000-0000-0000"
 	leaderTick    = time.Hour
-	healTick      = time.Hour
 	healInterval  = 30 * 24 * time.Hour
 )
 
@@ -75,6 +74,7 @@ func getLocalBackgroundHealStatus() madmin.BgHealState {
 	return madmin.BgHealState{
 		ScannedItemsCount: bgSeq.scannedItemsCount,
 		LastHealActivity:  bgSeq.lastHealActivity,
+		NextHealRound:     UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
 	}
 }
 
@@ -112,6 +112,19 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
 	return nil
 }
 
+// Returns the duration to the next background healing round
+func durationToNextHealRound(lastHeal time.Time) time.Duration {
+	if lastHeal.IsZero() {
+		lastHeal = globalBootTime
+	}
+
+	d := lastHeal.Add(healInterval).Sub(UTCNow())
+	if d < 0 {
+		return time.Second
+	}
+	return d
+}
+
 // Healing leader will take the charge of healing all erasure sets
 func execLeaderTasks(z *xlZones) {
 	ctx := context.Background()
@@ -132,10 +145,7 @@ func execLeaderTasks(z *xlZones) {
 
 	lastScanTime := time.Now() // So that we don't heal immediately, but after one month.
 	for {
-		if time.Since(lastScanTime) < healInterval {
+		time.Sleep(durationToNextHealRound(lastScanTime))
-			time.Sleep(healTick)
-			continue
-		}
 		for _, zone := range z.zones {
 			// Heal set by set
 			for i, set := range zone.sets {

pkg/madmin/heal-commands.go

@@ -274,6 +274,7 @@ func (adm *AdminClient) Heal(bucket, prefix string, healOpts HealOpts,
 type BgHealState struct {
 	ScannedItemsCount int64
 	LastHealActivity  time.Time
+	NextHealRound     time.Time
 }
 
 // BackgroundHealStatus returns the background heal status of the