From 745a4b31ba3c0bd6a040aba32c4ef348d45f2a45 Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Fri, 29 Jan 2021 21:59:49 -0800
Subject: [PATCH] add support for concurrent heals

---
 cmd/admin-handlers.go      | 28 ++++++++++++++++++++--------
 cmd/admin-heal-ops.go      |  3 ++-
 cmd/background-heal-ops.go |  3 ++-
 cmd/global-heal.go         |  5 +++--
 4 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go
index 6b3cb92f8..b14b26f72 100644
--- a/cmd/admin-handlers.go
+++ b/cmd/admin-handlers.go
@@ -32,6 +32,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -772,14 +773,25 @@ func (a adminAPIHandlers) HealSetsHandler(w http.ResponseWriter, r *http.Request
 
 	buckets, _ := objectAPI.ListBucketsHeal(ctx)
 	ctx, opts.cancel = context.WithCancel(context.Background())
-	for _, setNumber := range opts.setNumbers {
-		go func(setNumber int) {
-			lbDisks := z.serverSets[0].sets[setNumber].getOnlineDisks()
-			if err := healErasureSet(ctx, setNumber, opts.sleepForIO, opts.sleepDuration, buckets, lbDisks); err != nil {
-				logger.LogIf(ctx, err)
-			}
-		}(setNumber)
-	}
+	go func() {
+		var wg sync.WaitGroup
+		for _, setNumber := range opts.setNumbers {
+			wg.Add(1)
+			go func(setNumber int) {
+				defer wg.Done()
+				lbDisks := z.serverSets[0].sets[setNumber].getOnlineDisks()
+				if err := healErasureSet(ctx, setNumber, opts.sleepForIO, opts.sleepDuration, buckets, lbDisks); err != nil {
+					logger.LogIf(ctx, err)
+				}
+			}(setNumber)
+		}
+		wg.Wait()
+		a.mu.Lock()
+		opts.cancel()
+		delete(a.healSetsMap, opts.taskUUID)
+		a.mu.Unlock()
+		logger.Info("Healing finished for %v", vars[healSetsList])
+	}()
 
 	a.mu.Lock()
 	a.healSetsMap[opts.taskUUID] = opts
diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go
index 85e145795..604e2f0aa 100644
--- a/cmd/admin-heal-ops.go
+++ b/cmd/admin-heal-ops.go
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"runtime"
 	"sync"
 	"time"
 
@@ -420,7 +421,7 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 	clientToken := mustGetUUID()
 
 	return &healSequence{
-		respCh:         make(chan healResult),
+		respCh:         make(chan healResult, runtime.GOMAXPROCS(0)),
 		bucket:         bucket,
 		object:         objPrefix,
 		reportProgress: true,
diff --git a/cmd/background-heal-ops.go b/cmd/background-heal-ops.go
index 24b86e307..4505bead5 100644
--- a/cmd/background-heal-ops.go
+++ b/cmd/background-heal-ops.go
@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"path"
+	"runtime"
 	"time"
 
 	"github.com/minio/minio/pkg/madmin"
@@ -125,7 +126,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 
 func newHealRoutine() *healRoutine {
 	return &healRoutine{
-		tasks:  make(chan healTask),
+		tasks:  make(chan healTask, runtime.GOMAXPROCS(0)),
 		doneCh: make(chan struct{}),
 	}
 
diff --git a/cmd/global-heal.go b/cmd/global-heal.go
index d105d48fe..2d2973fb5 100644
--- a/cmd/global-heal.go
+++ b/cmd/global-heal.go
@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"runtime"
 	"sync"
 	"time"
 
@@ -42,8 +43,8 @@ func newBgHealSequence() *healSequence {
 	}
 
 	return &healSequence{
-		sourceCh:    make(chan healSource),
-		respCh:      make(chan healResult),
+		sourceCh:    make(chan healSource, runtime.GOMAXPROCS(0)),
+		respCh:      make(chan healResult, runtime.GOMAXPROCS(0)),
 		startTime:   UTCNow(),
 		clientToken: bgHealingUUID,
 		// run-background heal with reserved bucket