Move admin APIs to new path and add redesigned heal APIs (#5351)

- Changes related to moving admin APIs
   - admin APIs now have an endpoint under /minio/admin
   - admin APIs are now versioned - a new API to server the version is
     added at "GET /minio/admin/version" and all API operations have the
     path prefix /minio/admin/v1/<operation>
   - new service stop API added
   - credentials change API is moved to /minio/admin/v1/config/credential
   - credentials change API and configuration get/set API now require TLS
     so that credentials are protected
   - all API requests now receive JSON
   - heal APIs are disabled as they will be changed substantially

- Heal API changes
   Heal API is now provided at a single endpoint with the ability for a
   client to start a heal sequence on all the data in the server, a
   single bucket, or under a prefix within a bucket.

   When a heal sequence is started, the server returns a unique token
   that needs to be used for subsequent 'status' requests to fetch heal
   results.

   On each status request from the client, the server returns heal result
   records that it has accumulated since the previous status request. The
   server accumulates upto 1000 records and pauses healing further
   objects until the client requests for status. If the client does not
   request any further records for a long time, the server aborts the
   heal sequence automatically.

   A heal result record is returned for each entity healed on the server,
   such as system metadata, object metadata, buckets and objects, and has
   information about the before and after states on each disk.

   A client may request to force restart a heal sequence - this causes
   the running heal sequence to be aborted at the next safe spot and
   starts a new heal sequence.

cmd/format-xl.go

@@ -143,14 +143,14 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatXLV1, []error) {
 	// Initialize format configs.
 	var formats = make([]*formatXLV1, len(bootstrapDisks))
 
-	// Make a volume entry on all underlying storage disks.
+	// Load format from each disk in parallel
 	for index, disk := range bootstrapDisks {
 		if disk == nil {
 			sErrs[index] = errDiskNotFound
 			continue
 		}
 		wg.Add(1)
-		// Make a volume inside a go-routine.
+		// Launch go-routine per disk.
 		go func(index int, disk StorageAPI) {
 			defer wg.Done()
 			format, lErr := loadFormat(disk)
@@ -162,15 +162,9 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatXLV1, []error) {
 		}(index, disk)
 	}
 
-	// Wait for all make vol to finish.
+	// Wait for all go-routines to finish.
 	wg.Wait()
 
-	for _, err := range sErrs {
-		if err != nil {
-			// Return all formats and errors.
-			return formats, sErrs
-		}
-	}
 	// Return all formats and nil
 	return formats, sErrs
 }
@@ -374,8 +368,10 @@ func loadFormat(disk StorageAPI) (format *formatXLV1, err error) {
 			if err != nil {
 				return nil, err
 			}
-			if len(vols) > 1 {
-				// 'format.json' not found, but we found user data.
+			if len(vols) > 1 || (len(vols) == 1 &&
+				vols[0].Name != minioMetaBucket) {
+				// 'format.json' not found, but we
+				// found user data.
 				return nil, errCorruptedFormat
 			}
 			// No other data found, its a fresh disk.
@@ -394,11 +390,10 @@ func loadFormat(disk StorageAPI) (format *formatXLV1, err error) {
 	return format, nil
 }
 
-// collectNSaveNewFormatConfigs - creates new format configs based on
-// the reference config and saves it on all disks, this is to be
-// called from healFormatXL* functions.
+// collectNSaveNewFormatConfigs - generates new format configs based on
+// the given ref. config and saves on each disk
 func collectNSaveNewFormatConfigs(referenceConfig *formatXLV1,
-	orderedDisks []StorageAPI) error {
+	orderedDisks []StorageAPI, dryRun bool) error {
 
 	// Collect new format configs that need to be written.
 	var newFormatConfigs = make([]*formatXLV1, len(orderedDisks))
@@ -420,16 +415,19 @@ func collectNSaveNewFormatConfigs(referenceConfig *formatXLV1,
 	}
 
 	// Save new `format.json` across all disks, in JBOD order.
-	return saveFormatXL(orderedDisks, newFormatConfigs)
+	if !dryRun {
+		return saveFormatXL(orderedDisks, newFormatConfigs)
+	}
+	return nil
 }
 
 // Heals any missing format.json on the drives. Returns error only for
 // unexpected errors as regular errors can be ignored since there
 // might be enough quorum to be operational.  Heals only fresh disks.
 func healFormatXLFreshDisks(storageDisks []StorageAPI,
-	formats []*formatXLV1) error {
+	formats []*formatXLV1, dryRun bool) error {
 
-	// Reorder the disks based on the JBOD order.
+	// Reorder disks based on JBOD order, and get reference config.
 	referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
 		formats, true)
 	if err != nil {
@@ -441,21 +439,24 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI,
 	// and allowed fresh disks to be arranged anywhere.
 	// Following block facilitates to put fresh disks.
 	for index, format := range formats {
+		if format != nil {
+			continue
+		}
+
 		// Format is missing so we go through ordered disks.
-		if format == nil {
-			// At this point when disk is missing the fresh disk
-			// in the stack get it back from storageDisks.
-			for oIndex, disk := range orderedDisks {
-				if disk == nil {
-					orderedDisks[oIndex] = storageDisks[index]
-					break
-				}
+		// At this point when disk is missing the fresh disk
+		// in the stack get it back from storageDisks.
+		for oIndex, disk := range orderedDisks {
+			if disk == nil {
+				orderedDisks[oIndex] = storageDisks[index]
+				break
 			}
 		}
 	}
 
 	// apply new format config and save to all disks
-	return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
+	return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks,
+		dryRun)
 }
 
 // collectUnAssignedDisks - collect disks unassigned to orderedDisks
@@ -540,9 +541,9 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
 
 // Heals corrupted format json in all disks
 func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
-	formats []*formatXLV1) error {
+	formats []*formatXLV1, dryRun bool) error {
 
-	// Reorder the disks based on the JBOD order.
+	// Reorder disks based on JBOD order, and update ref. config.
 	referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
 		formats, true)
 	if err != nil {
@@ -570,8 +571,9 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
 		}
 	}
 
-	// apply new format config and save to all disks
-	return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
+	// generate and write new configs to all disks
+	return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks,
+		dryRun)
 }
 
 // loadFormatXL - loads XL `format.json` and returns back properly