Imporve healing and reporting (#11312)

* Provide information on *actively* healing, buckets healed/queued, objects healed/failed.
* Add concurrent healing of multiple sets (typically on startup).
* Add bucket level resume, so restarts will only heal non-healed buckets.
* Print summary after healing a disk is done.

cmd/erasure-metadata-utils_test.go

@@ -18,6 +18,9 @@ package cmd
 
 import (
 	"context"
+	"encoding/hex"
+	"fmt"
+	"math/rand"
 	"reflect"
 	"testing"
 )
@@ -199,3 +202,22 @@ func TestEvalDisks(t *testing.T) {
 	z := objLayer.(*erasureServerPools)
 	testShuffleDisks(t, z)
 }
+
+func Test_hashOrder(t *testing.T) {
+	for x := 1; x < 17; x++ {
+		t.Run(fmt.Sprintf("%d", x), func(t *testing.T) {
+			var first [17]int
+			rng := rand.New(rand.NewSource(0))
+			var tmp [16]byte
+			rng.Read(tmp[:])
+			prefix := hex.EncodeToString(tmp[:])
+			for i := 0; i < 10000; i++ {
+				rng.Read(tmp[:])
+
+				y := hashOrder(fmt.Sprintf("%s/%x", prefix, hex.EncodeToString(tmp[:3])), x)
+				first[y[0]]++
+			}
+			t.Log("first:", first[:x])
+		})
+	}
+}