fix: make sure to correctly initialize health checks (#17765)

health checks were missing for drives replaced since

- HealFormat() would replace the drives without a health check
- disconnected drives when they reconnect via connectEndpoint()
  the loop also loses health checks for local disks and merges
  these into a single code.
- other than this separate cleanUp, health check variables to avoid
  overloading them with similar requirements.
- also ensure that we compete via context selector for disk monitoring
  such that the canceled disks don't linger around longer waiting for
  the ticker to trigger.
- allow disabling active monitoring.

cmd/prepare-storage.go

@@ -101,11 +101,12 @@ func bgFormatErasureCleanupTmp(diskPath string) {
 	// Delete all temporary files created for DirectIO write check
 	files, _ := filepath.Glob(filepath.Join(diskPath, ".writable-check-*.tmp"))
 	for _, file := range files {
-		removeAll(file)
+		go removeAll(file)
 	}
 
 	// Remove the entire folder in case there are leftovers that didn't get cleaned up before restart.
 	go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old"))
+
 	// Renames and schedules for purging all bucket metacache.
 	go renameAllBucketMetacache(diskPath)
 }
@@ -155,7 +156,7 @@ func isServerResolvable(endpoint Endpoint, timeout time.Duration) error {
 // time. additionally make sure to close all the disks used in this attempt.
 func connectLoadInitFormats(verboseLogging bool, firstDisk bool, endpoints Endpoints, poolCount, setCount, setDriveCount int, deploymentID, distributionAlgo string) (storageDisks []StorageAPI, format *formatErasureV3, err error) {
 	// Initialize all storage disks
-	storageDisks, errs := initStorageDisksWithErrors(endpoints, true)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints, storageOpts{cleanUp: true, healthCheck: true})
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {