Add self-healing feature (#7604)

- Background Heal routine receives heal requests from a channel, either to
heal format, buckets or objects
- Daily sweeper lists all objects in all buckets, these objects
don't necessarly have read quorum so they can be removed if
these objects are unhealable
- Heal daily ops receives objects from the daily sweeper
and send them to the heal routine.

cmd/server-main.go

@@ -287,8 +287,11 @@ func serverMain(ctx *cli.Context) {
 	// Initialize name space lock.
 	initNSLock(globalIsDistXL)
 
-	// Init global heal state
-	initAllHealState(globalIsXL)
+	if globalIsXL {
+		// Init global heal state
+		globalAllHealState = initHealState()
+		globalSweepHealState = initHealState()
+	}
 
 	// initialize globalTrace system
 	globalTrace = NewTraceSys(context.Background(), globalEndpoints)
@@ -372,6 +375,12 @@ func serverMain(ctx *cli.Context) {
 		logger.Fatal(errors.New("Invalid KMS configuration"), "auto-encryption is enabled but server does not support encryption")
 	}
 
+	if globalIsXL {
+		initBackgroundHealing()
+		initDailyHeal()
+		initDailySweeper()
+	}
+
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()