fix: reduce the load on CPU when loading users/policies (#8984)

Trying to be conservative by slowing ourselves down
on a regular basis.

cmd/background-heal-ops.go

@@ -52,6 +52,19 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
+func waitForLowHTTPReq(tolerance int32) {
+	if httpServer := newHTTPServerFn(); httpServer != nil {
+		// Wait at max 10 minute for an inprogress request before proceeding to heal
+		waitCount := 600
+		// Any requests in progress, delay the heal.
+		for (httpServer.GetRequestCount() >= tolerance) &&
+			waitCount > 0 {
+			waitCount--
+			time.Sleep(1 * time.Second)
+		}
+	}
+}
+
 // Wait for heal requests and process them
 func (h *healRoutine) run() {
 	ctx := context.Background()
@@ -62,16 +75,7 @@ func (h *healRoutine) run() {
 				break
 			}
 
-			if httpServer := newHTTPServerFn(); httpServer != nil {
+			waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
-				// Wait at max 10 minute for an inprogress request before proceeding to heal
-				waitCount := 600
-				// Any requests in progress, delay the heal.
-				for (httpServer.GetRequestCount() >= int32(globalEndpoints.Nodes())) &&
-					waitCount > 0 {
-					waitCount--
-					time.Sleep(1 * time.Second)
-				}
-			}
 
 			var res madmin.HealResultItem
 			var err error

cmd/iam-object-store.go

@@ -579,6 +579,8 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 	}
 
 	go func() {
+		defer close(ch)
+
 		marker := ""
 		for {
 			lo, err := objectAPI.ListObjects(context.Background(),
@@ -588,9 +590,13 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 				case ch <- itemOrErr{Err: err}:
 				case <-doneCh:
 				}
-				close(ch)
 				return
 			}
+
+			// Slow down listing and loading for config items to
+			// reduce load on the server
+			waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
+
 			marker = lo.NextMarker
 			lister := dirList(lo)
 			if !dirs {
@@ -602,12 +608,10 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 				select {
 				case ch <- itemOrErr{Item: item}:
 				case <-doneCh:
-					close(ch)
 					return
 				}
 			}
 			if !lo.IsTruncated {
-				close(ch)
 				return
 			}
 		}
@@ -617,17 +621,17 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 
 func (iamOS *IAMObjectStore) watch(sys *IAMSys) {
 	watchDisk := func() {
-		ticker := time.NewTicker(globalRefreshIAMInterval)
-		defer ticker.Stop()
 		for {
 			select {
 			case <-GlobalServiceDoneCh:
 				return
-			case <-ticker.C:
+			case <-time.NewTimer(globalRefreshIAMInterval).C:
-				iamOS.loadAll(sys, nil)
+				err := iamOS.loadAll(sys, nil)
+				logger.LogIf(context.Background(), err)
 			}
 		}
 	}
+
 	// Refresh IAMSys in background.
 	go watchDisk()
 }

cmd/xl-sets.go

@@ -1664,16 +1664,7 @@ func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObj
 			continue
 		}
 
-		if httpServer := newHTTPServerFn(); httpServer != nil {
+		waitForLowHTTPReq(int32(s.drivesPerSet))
-			// Wait at max 10 minute for an inprogress request before proceeding to heal
-			waitCount := 600
-			// Any requests in progress, delay the heal.
-			for (httpServer.GetRequestCount() >= int32(s.drivesPerSet)) &&
-				waitCount > 0 {
-				waitCount--
-				time.Sleep(1 * time.Second)
-			}
-		}
 
 		if err := healObject(bucket, entry.Name); err != nil {
 			return toObjectErr(err, bucket, entry.Name)

cmd/xl-zones.go

@@ -24,7 +24,6 @@ import (
 	"net/http"
 	"strings"
 	"sync"
-	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -1364,16 +1363,7 @@ func (z *xlZones) HealObjects(ctx context.Context, bucket, prefix string, healOb
 			continue
 		}
 
-		if httpServer := newHTTPServerFn(); httpServer != nil {
+		waitForLowHTTPReq(int32(zoneDrivesPerSet[zoneIndex]))
-			// Wait at max 10 minute for an inprogress request before proceeding to heal
-			waitCount := 600
-			// Any requests in progress, delay the heal.
-			for (httpServer.GetRequestCount() >= int32(zoneDrivesPerSet[zoneIndex])) &&
-				waitCount > 0 {
-				waitCount--
-				time.Sleep(1 * time.Second)
-			}
-		}
 
 		if err := healObject(bucket, entry.Name); err != nil {
 			return toObjectErr(err, bucket, entry.Name)