diff --git a/cmd/background-heal-ops.go b/cmd/background-heal-ops.go index 946530e0c..bce2efda6 100644 --- a/cmd/background-heal-ops.go +++ b/cmd/background-heal-ops.go @@ -52,6 +52,19 @@ func (h *healRoutine) queueHealTask(task healTask) { h.tasks <- task } +func waitForLowHTTPReq(tolerance int32) { + if httpServer := newHTTPServerFn(); httpServer != nil { + // Wait at max 10 minute for an inprogress request before proceeding to heal + waitCount := 600 + // Any requests in progress, delay the heal. + for (httpServer.GetRequestCount() >= tolerance) && + waitCount > 0 { + waitCount-- + time.Sleep(1 * time.Second) + } + } +} + // Wait for heal requests and process them func (h *healRoutine) run() { ctx := context.Background() @@ -62,16 +75,7 @@ func (h *healRoutine) run() { break } - if httpServer := newHTTPServerFn(); httpServer != nil { - // Wait at max 10 minute for an inprogress request before proceeding to heal - waitCount := 600 - // Any requests in progress, delay the heal. - for (httpServer.GetRequestCount() >= int32(globalEndpoints.Nodes())) && - waitCount > 0 { - waitCount-- - time.Sleep(1 * time.Second) - } - } + waitForLowHTTPReq(int32(globalEndpoints.Nodes())) var res madmin.HealResultItem var err error diff --git a/cmd/iam-object-store.go b/cmd/iam-object-store.go index 5d18844d3..aa5944b14 100644 --- a/cmd/iam-object-store.go +++ b/cmd/iam-object-store.go @@ -579,6 +579,8 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool, } go func() { + defer close(ch) + marker := "" for { lo, err := objectAPI.ListObjects(context.Background(), @@ -588,9 +590,13 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool, case ch <- itemOrErr{Err: err}: case <-doneCh: } - close(ch) return } + + // Slow down listing and loading for config items to + // reduce load on the server + waitForLowHTTPReq(int32(globalEndpoints.Nodes())) + marker = lo.NextMarker lister := dirList(lo) if !dirs { @@ -602,12 +608,10 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool, select { case ch <- itemOrErr{Item: item}: case <-doneCh: - close(ch) return } } if !lo.IsTruncated { - close(ch) return } } @@ -617,17 +621,17 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool, func (iamOS *IAMObjectStore) watch(sys *IAMSys) { watchDisk := func() { - ticker := time.NewTicker(globalRefreshIAMInterval) - defer ticker.Stop() for { select { case <-GlobalServiceDoneCh: return - case <-ticker.C: - iamOS.loadAll(sys, nil) + case <-time.NewTimer(globalRefreshIAMInterval).C: + err := iamOS.loadAll(sys, nil) + logger.LogIf(context.Background(), err) } } } + // Refresh IAMSys in background. go watchDisk() } diff --git a/cmd/xl-sets.go b/cmd/xl-sets.go index fd4ceba3f..71df46912 100644 --- a/cmd/xl-sets.go +++ b/cmd/xl-sets.go @@ -1664,16 +1664,7 @@ func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObj continue } - if httpServer := newHTTPServerFn(); httpServer != nil { - // Wait at max 10 minute for an inprogress request before proceeding to heal - waitCount := 600 - // Any requests in progress, delay the heal. - for (httpServer.GetRequestCount() >= int32(s.drivesPerSet)) && - waitCount > 0 { - waitCount-- - time.Sleep(1 * time.Second) - } - } + waitForLowHTTPReq(int32(s.drivesPerSet)) if err := healObject(bucket, entry.Name); err != nil { return toObjectErr(err, bucket, entry.Name) diff --git a/cmd/xl-zones.go b/cmd/xl-zones.go index a3ab71f7b..3d0736919 100644 --- a/cmd/xl-zones.go +++ b/cmd/xl-zones.go @@ -24,7 +24,6 @@ import ( "net/http" "strings" "sync" - "time" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" @@ -1364,16 +1363,7 @@ func (z *xlZones) HealObjects(ctx context.Context, bucket, prefix string, healOb continue } - if httpServer := newHTTPServerFn(); httpServer != nil { - // Wait at max 10 minute for an inprogress request before proceeding to heal - waitCount := 600 - // Any requests in progress, delay the heal. - for (httpServer.GetRequestCount() >= int32(zoneDrivesPerSet[zoneIndex])) && - waitCount > 0 { - waitCount-- - time.Sleep(1 * time.Second) - } - } + waitForLowHTTPReq(int32(zoneDrivesPerSet[zoneIndex])) if err := healObject(bucket, entry.Name); err != nil { return toObjectErr(err, bucket, entry.Name)