Move admin APIs to new path and add redesigned heal APIs (#5351)

- Changes related to moving admin APIs
   - admin APIs now have an endpoint under /minio/admin
   - admin APIs are now versioned - a new API to server the version is
     added at "GET /minio/admin/version" and all API operations have the
     path prefix /minio/admin/v1/<operation>
   - new service stop API added
   - credentials change API is moved to /minio/admin/v1/config/credential
   - credentials change API and configuration get/set API now require TLS
     so that credentials are protected
   - all API requests now receive JSON
   - heal APIs are disabled as they will be changed substantially

- Heal API changes
   Heal API is now provided at a single endpoint with the ability for a
   client to start a heal sequence on all the data in the server, a
   single bucket, or under a prefix within a bucket.

   When a heal sequence is started, the server returns a unique token
   that needs to be used for subsequent 'status' requests to fetch heal
   results.

   On each status request from the client, the server returns heal result
   records that it has accumulated since the previous status request. The
   server accumulates upto 1000 records and pauses healing further
   objects until the client requests for status. If the client does not
   request any further records for a long time, the server aborts the
   heal sequence automatically.

   A heal result record is returned for each entity healed on the server,
   such as system metadata, object metadata, buckets and objects, and has
   information about the before and after states on each disk.

   A client may request to force restart a heal sequence - this causes
   the running heal sequence to be aborted at the next safe spot and
   starts a new heal sequence.

cmd/xl-v1-healing.go

@@ -23,11 +23,14 @@ import (
 	"sync"
 
 	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // healFormatXL - heals missing `format.json` on freshly or corrupted
 // disks (missing format.json but does have erasure coded data in it).
-func healFormatXL(storageDisks []StorageAPI) (err error) {
+func healFormatXL(storageDisks []StorageAPI, dryRun bool) (res madmin.HealResultItem,
+	err error) {
+
 	// Attempt to load all `format.json`.
 	formatConfigs, sErrs := loadAllFormats(storageDisks)
 
@@ -35,7 +38,34 @@ func healFormatXL(storageDisks []StorageAPI) (err error) {
 	// - if (no quorum) return error
 	// - if (disks not recognized) // Always error.
 	if err = genericFormatCheckXL(formatConfigs, sErrs); err != nil {
-		return err
+		return res, err
+	}
+
+	// Prepare heal-result
+	res = madmin.HealResultItem{
+		Type:      madmin.HealItemMetadata,
+		Detail:    "disk-format",
+		DiskCount: len(storageDisks),
+	}
+	res.InitDrives()
+	// Existing formats are available (i.e. ok), so save it in
+	// result, also populate disks to be healed.
+	for i, format := range formatConfigs {
+		drive := globalEndpoints.GetString(i)
+		switch {
+		case format != nil:
+			res.DriveInfo.Before[drive] = madmin.DriveStateOk
+		case sErrs[i] == errCorruptedFormat:
+			res.DriveInfo.Before[drive] = madmin.DriveStateCorrupt
+		case sErrs[i] == errUnformattedDisk:
+			res.DriveInfo.Before[drive] = madmin.DriveStateMissing
+		default:
+			res.DriveInfo.Before[drive] = madmin.DriveStateOffline
+		}
+	}
+	// Copy "after" drive state too
+	for k, v := range res.DriveInfo.Before {
+		res.DriveInfo.After[k] = v
 	}
 
 	numDisks := len(storageDisks)
@@ -45,61 +75,95 @@ func healFormatXL(storageDisks []StorageAPI) (err error) {
 	switch {
 	case unformattedDiskCount == numDisks:
 		// all unformatted.
-		if err = initFormatXL(storageDisks); err != nil {
-			return err
+		if !dryRun {
+			err = initFormatXL(storageDisks)
+			if err != nil {
+				return res, err
+			}
+			for i := 0; i < len(storageDisks); i++ {
+				drive := globalEndpoints.GetString(i)
+				res.DriveInfo.After[drive] = madmin.DriveStateOk
+			}
 		}
+		return res, nil
 
 	case diskNotFoundCount > 0:
-		return fmt.Errorf("cannot proceed with heal as %s",
+		return res, fmt.Errorf("cannot proceed with heal as %s",
 			errSomeDiskOffline)
 
 	case otherErrCount > 0:
-		return fmt.Errorf("cannot proceed with heal as some disks had unhandled errors")
+		return res, fmt.Errorf("cannot proceed with heal as some disks had unhandled errors")
 
 	case corruptedFormatCount > 0:
-		if err = healFormatXLCorruptedDisks(storageDisks, formatConfigs); err != nil {
-			return fmt.Errorf("Unable to repair corrupted format, %s", err)
+		// heal corrupted disks
+		err = healFormatXLCorruptedDisks(storageDisks, formatConfigs,
+			dryRun)
+		if err != nil {
+			return res, err
 		}
+		// success
+		if !dryRun {
+			for i := 0; i < len(storageDisks); i++ {
+				drive := globalEndpoints.GetString(i)
+				res.DriveInfo.After[drive] = madmin.DriveStateOk
+			}
+		}
+		return res, nil
 
 	case unformattedDiskCount > 0:
-		// All drives online but some report missing format.json.
-		if err = healFormatXLFreshDisks(storageDisks, formatConfigs); err != nil {
-			// There was an unexpected unrecoverable error
-			// during healing.
-			return fmt.Errorf("Unable to heal backend %s", err)
+		// heal unformatted disks
+		err = healFormatXLFreshDisks(storageDisks, formatConfigs,
+			dryRun)
+		if err != nil {
+			return res, err
 		}
-
+		// success
+		if !dryRun {
+			for i := 0; i < len(storageDisks); i++ {
+				drive := globalEndpoints.GetString(i)
+				res.DriveInfo.After[drive] = madmin.DriveStateOk
+			}
+		}
+		return res, nil
 	}
-	return nil
+
+	return res, nil
 }
 
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
-func (xl xlObjects) HealBucket(bucket string) error {
-	if err := checkBucketExist(bucket, xl); err != nil {
-		return err
+func (xl xlObjects) HealBucket(bucket string, dryRun bool) (
+	results []madmin.HealResultItem, err error) {
+
+	if err = checkBucketExist(bucket, xl); err != nil {
+		return nil, err
 	}
 
 	// get write quorum for an object
 	writeQuorum := len(xl.storageDisks)/2 + 1
 	bucketLock := xl.nsMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalHealingTimeout); err != nil {
-		return err
+	if err = bucketLock.GetLock(globalHealingTimeout); err != nil {
+		return nil, err
 	}
 	defer bucketLock.Unlock()
 
 	// Heal bucket.
-	if err := healBucket(xl.storageDisks, bucket, writeQuorum); err != nil {
-		return err
+	result, err := healBucket(xl.storageDisks, bucket, writeQuorum, dryRun)
+	if err != nil {
+		return results, err
 	}
+	results = append(results, result)
 
 	// Proceed to heal bucket metadata.
-	return healBucketMetadata(xl, bucket)
+	metaResults, err := healBucketMetadata(xl, bucket, dryRun)
+	results = append(results, metaResults...)
+	return results, err
 }
 
 // Heal bucket - create buckets on disks where it does not exist.
-func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int) error {
+func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int,
+	dryRun bool) (res madmin.HealResultItem, err error) {
 
 	// Initialize sync waitgroup.
 	var wg = &sync.WaitGroup{}
@@ -107,24 +171,47 @@ func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int) error
 	// Initialize list of errors.
 	var dErrs = make([]error, len(storageDisks))
 
+	// Disk states slices
+	beforeState := make([]string, len(storageDisks))
+	afterState := make([]string, len(storageDisks))
+
 	// Make a volume entry on all underlying storage disks.
 	for index, disk := range storageDisks {
 		if disk == nil {
 			dErrs[index] = errors.Trace(errDiskNotFound)
+			beforeState[index] = madmin.DriveStateOffline
+			afterState[index] = madmin.DriveStateOffline
 			continue
 		}
 		wg.Add(1)
+
 		// Make a volume inside a go-routine.
 		go func(index int, disk StorageAPI) {
 			defer wg.Done()
 			if _, err := disk.StatVol(bucket); err != nil {
-				if err != errVolumeNotFound {
+				if errors.Cause(err) != errVolumeNotFound {
+					beforeState[index] = madmin.DriveStateCorrupt
+					afterState[index] = madmin.DriveStateCorrupt
 					dErrs[index] = errors.Trace(err)
 					return
 				}
-				if err = disk.MakeVol(bucket); err != nil {
-					dErrs[index] = errors.Trace(err)
+
+				beforeState[index] = madmin.DriveStateMissing
+				afterState[index] = madmin.DriveStateMissing
+
+				// mutate only if not a dry-run
+				if dryRun {
+					return
 				}
+
+				makeErr := disk.MakeVol(bucket)
+				dErrs[index] = errors.Trace(makeErr)
+				if makeErr == nil {
+					afterState[index] = madmin.DriveStateOk
+				}
+			} else {
+				beforeState[index] = madmin.DriveStateOk
+				afterState[index] = madmin.DriveStateOk
 			}
 		}(index, disk)
 	}
@@ -132,44 +219,75 @@ func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int) error
 	// Wait for all make vol to finish.
 	wg.Wait()
 
+	// Initialize heal result info
+	res = madmin.HealResultItem{
+		Type:      madmin.HealItemBucket,
+		Bucket:    bucket,
+		DiskCount: len(storageDisks),
+	}
+	res.InitDrives()
+	for i, before := range beforeState {
+		drive := globalEndpoints.GetString(i)
+		res.DriveInfo.Before[drive] = before
+		res.DriveInfo.After[drive] = afterState[i]
+	}
+
 	reducedErr := reduceWriteQuorumErrs(dErrs, bucketOpIgnoredErrs, writeQuorum)
 	if errors.Cause(reducedErr) == errXLWriteQuorum {
 		// Purge successfully created buckets if we don't have writeQuorum.
 		undoMakeBucket(storageDisks, bucket)
 	}
-	return reducedErr
+	return res, reducedErr
 }
 
 // Heals all the metadata associated for a given bucket, this function
 // heals `policy.json`, `notification.xml` and `listeners.json`.
-func healBucketMetadata(xlObj xlObjects, bucket string) error {
+func healBucketMetadata(xl xlObjects, bucket string, dryRun bool) (
+	results []madmin.HealResultItem, err error) {
+
 	healBucketMetaFn := func(metaPath string) error {
-		if _, _, err := xlObj.HealObject(minioMetaBucket, metaPath); err != nil && !isErrObjectNotFound(err) {
-			return err
+		result, healErr := xl.HealObject(minioMetaBucket, metaPath, dryRun)
+		// If object is not found, no result to add.
+		if isErrObjectNotFound(healErr) {
+			return nil
 		}
+		if healErr != nil {
+			return healErr
+		}
+		result.Type = madmin.HealItemBucketMetadata
+		results = append(results, result)
 		return nil
 	}
 
-	// Heal `policy.json` for missing entries, ignores if `policy.json` is not found.
+	// Heal `policy.json` for missing entries, ignores if
+	// `policy.json` is not found.
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	if err := healBucketMetaFn(policyPath); err != nil {
-		return err
+	err = healBucketMetaFn(policyPath)
+	if err != nil {
+		return results, err
 	}
 
-	// Heal `notification.xml` for missing entries, ignores if `notification.xml` is not found.
-	nConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
-	if err := healBucketMetaFn(nConfigPath); err != nil {
-		return err
+	// Heal `notification.xml` for missing entries, ignores if
+	// `notification.xml` is not found.
+	nConfigPath := path.Join(bucketConfigPrefix, bucket,
+		bucketNotificationConfig)
+	err = healBucketMetaFn(nConfigPath)
+	if err != nil {
+		return results, err
 	}
 
-	// Heal `listeners.json` for missing entries, ignores if `listeners.json` is not found.
+	// Heal `listeners.json` for missing entries, ignores if
+	// `listeners.json` is not found.
 	lConfigPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
-	return healBucketMetaFn(lConfigPath)
+	err = healBucketMetaFn(lConfigPath)
+	return results, err
 }
 
 // listAllBuckets lists all buckets from all disks. It also
 // returns the occurrence of each buckets in all disks
-func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo, bucketsOcc map[string]int, err error) {
+func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo,
+	bucketsOcc map[string]int, err error) {
+
 	buckets = make(map[string]VolInfo)
 	bucketsOcc = make(map[string]int)
 	for _, disk := range storageDisks {
@@ -178,122 +296,42 @@ func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo, buck
 		}
 		var volsInfo []VolInfo
 		volsInfo, err = disk.ListVols()
-		if err == nil {
-			for _, volInfo := range volsInfo {
-				// StorageAPI can send volume names which are
-				// incompatible with buckets, handle it and skip them.
-				if !IsValidBucketName(volInfo.Name) {
-					continue
-				}
-				// Skip special volume buckets.
-				if isMinioMetaBucketName(volInfo.Name) {
-					continue
-				}
-				// Increase counter per bucket name
-				bucketsOcc[volInfo.Name]++
-				// Save volume info under bucket name
-				buckets[volInfo.Name] = volInfo
+		if err != nil {
+			if errors.IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) {
+				continue
 			}
-			continue
+			break
 		}
-		// Ignore any disks not found.
-		if errors.IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) {
-			continue
+		for _, volInfo := range volsInfo {
+			// StorageAPI can send volume names which are
+			// incompatible with buckets - these are
+			// skipped, like the meta-bucket.
+			if !IsValidBucketName(volInfo.Name) ||
+				isMinioMetaBucketName(volInfo.Name) {
+				continue
+			}
+			// Increase counter per bucket name
+			bucketsOcc[volInfo.Name]++
+			// Save volume info under bucket name
+			buckets[volInfo.Name] = volInfo
 		}
-		break
 	}
 	return buckets, bucketsOcc, err
 }
 
-// reduceHealStatus - fetches the worst heal status in a provided slice
-func reduceHealStatus(status []healStatus) healStatus {
-	worstStatus := healthy
-	for _, st := range status {
-		if st > worstStatus {
-			worstStatus = st
-		}
-	}
-	return worstStatus
-}
-
-// bucketHealStatus - returns the heal status of the provided bucket. Internally,
-// this function lists all object heal status of objects inside meta bucket config
-// directory and returns the worst heal status that can be found
-func (xl xlObjects) bucketHealStatus(bucketName string) (healStatus, error) {
-	// A list of all the bucket config files
-	configFiles := []string{bucketPolicyConfig, bucketNotificationConfig, bucketListenerConfig}
-	// The status of buckets config files
-	configsHealStatus := make([]healStatus, len(configFiles))
-	// The list of errors found during checking heal status of each config file
-	configsErrs := make([]error, len(configFiles))
-	// The path of meta bucket that contains all config files
-	configBucket := path.Join(minioMetaBucket, bucketConfigPrefix, bucketName)
-
-	// Check of config files heal status in go-routines
-	var wg sync.WaitGroup
-	// Loop over config files
-	for idx, configFile := range configFiles {
-		wg.Add(1)
-		// Compute heal status of current config file
-		go func(bucket, object string, index int) {
-			defer wg.Done()
-			// Check
-			listObjectsHeal, err := xl.listObjectsHeal(bucket, object, "", "", 1)
-			// If any error, save and immediately quit
-			if err != nil {
-				configsErrs[index] = err
-				return
-			}
-			// Check if current bucket contains any not healthy config file and save heal status
-			if len(listObjectsHeal.Objects) > 0 {
-				configsHealStatus[index] = listObjectsHeal.Objects[0].HealObjectInfo.Status
-			}
-		}(configBucket, configFile, idx)
-	}
-	wg.Wait()
-
-	// Return any found error
-	for _, err := range configsErrs {
-		if err != nil {
-			return healthy, err
-		}
-	}
-
-	// Reduce and return heal status
-	return reduceHealStatus(configsHealStatus), nil
-}
-
 // ListBucketsHeal - Find all buckets that need to be healed
 func (xl xlObjects) ListBucketsHeal() ([]BucketInfo, error) {
 	listBuckets := []BucketInfo{}
 	// List all buckets that can be found in all disks
-	buckets, occ, err := listAllBuckets(xl.storageDisks)
+	buckets, _, err := listAllBuckets(xl.storageDisks)
 	if err != nil {
 		return listBuckets, err
 	}
 
 	// Iterate over all buckets
 	for _, currBucket := range buckets {
-		// Check the status of bucket metadata
-		bucketHealStatus, err := xl.bucketHealStatus(currBucket.Name)
-		if err != nil {
-			return []BucketInfo{}, err
-		}
-		// If all metadata are sane, check if the bucket directory is present in all disks
-		if bucketHealStatus == healthy && occ[currBucket.Name] != len(xl.storageDisks) {
-			// Current bucket is missing in some of the storage disks
-			bucketHealStatus = canHeal
-		}
-		// Add current bucket to the returned result if not healthy
-		if bucketHealStatus != healthy {
-			listBuckets = append(listBuckets,
-				BucketInfo{
-					Name:           currBucket.Name,
-					Created:        currBucket.Created,
-					HealBucketInfo: &HealBucketInfo{Status: bucketHealStatus},
-				})
-		}
-
+		listBuckets = append(listBuckets,
+			BucketInfo{currBucket.Name, currBucket.Created})
 	}
 
 	// Sort found buckets
@@ -323,8 +361,8 @@ func quickHeal(xlObj xlObjects, writeQuorum int, readQuorum int) error {
 			defer bucketLock.Unlock()
 
 			// Heal bucket and then proceed to heal bucket metadata if any.
-			if err = healBucket(xlObj.storageDisks, bucketName, writeQuorum); err == nil {
-				if err = healBucketMetadata(xlObj, bucketName); err == nil {
+			if _, err = healBucket(xlObj.storageDisks, bucketName, writeQuorum, false); err == nil {
+				if _, err = healBucketMetadata(xlObj, bucketName, false); err == nil {
 					continue
 				}
 				return err
@@ -337,77 +375,108 @@ func quickHeal(xlObj xlObjects, writeQuorum int, readQuorum int) error {
 	return nil
 }
 
-// Heals an object only the corrupted/missing erasure blocks.
-func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (int, int, error) {
+// Heals an object by re-writing corrupt/missing erasure blocks.
+func healObject(storageDisks []StorageAPI, bucket string, object string,
+	quorum int, dryRun bool) (result madmin.HealResultItem, err error) {
 
 	partsMetadata, errs := readAllXLMetadata(storageDisks, bucket, object)
+
 	// readQuorum suffices for xl.json since we use monotonic
 	// system time to break the tie when a split-brain situation
 	// arises.
-	if rErr := reduceReadQuorumErrs(errs, nil, quorum); rErr != nil {
-		return 0, 0, toObjectErr(rErr, bucket, object)
+	if reducedErr := reduceReadQuorumErrs(errs, nil, quorum); reducedErr != nil {
+		return result, toObjectErr(reducedErr, bucket, object)
 	}
 
-	// List of disks having latest version of the object.
+	// List of disks having latest version of the object xl.json
+	// (by modtime).
 	latestDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
-	// List of disks having all parts as per latest xl.json - this
-	// does a full pass over the data and verifies all part files
-	// on disk
-	availableDisks, errs, aErr := disksWithAllParts(latestDisks, partsMetadata, errs, bucket,
-		object)
+	// List of disks having all parts as per latest xl.json.
+	availableDisks, dataErrs, aErr := disksWithAllParts(latestDisks, partsMetadata, errs, bucket, object)
 	if aErr != nil {
-		return 0, 0, toObjectErr(aErr, bucket, object)
+		return result, toObjectErr(aErr, bucket, object)
 	}
 
-	// Number of disks which don't serve data.
-	numOfflineDisks := 0
-	for index, disk := range storageDisks {
-		if disk == nil || errs[index] == errDiskNotFound {
-			numOfflineDisks++
-		}
-	}
+	// Initialize heal result object
+	result = madmin.HealResultItem{
+		Type:      madmin.HealItemObject,
+		Bucket:    bucket,
+		Object:    object,
+		DiskCount: len(storageDisks),
 
-	// Number of disks which have all parts of the given object.
+		// Initialize object size to -1, so we can detect if we are
+		// unable to reliably find the object size.
+		ObjectSize: -1,
+	}
+	result.InitDrives()
+
+	// Loop to find number of disks with valid data, per-drive
+	// data state and a list of outdated disks on which data needs
+	// to be healed.
+	outDatedDisks := make([]StorageAPI, len(storageDisks))
 	numAvailableDisks := 0
-	for _, disk := range availableDisks {
-		if disk != nil {
+	disksToHealCount := 0
+	for i, v := range availableDisks {
+		driveState := ""
+		switch {
+		case v != nil:
+			driveState = madmin.DriveStateOk
 			numAvailableDisks++
+			// If data is sane on any one disk, we can
+			// extract the correct object size.
+			result.ObjectSize = partsMetadata[i].Stat.Size
+			result.ParityBlocks = partsMetadata[i].Erasure.ParityBlocks
+			result.DataBlocks = partsMetadata[i].Erasure.DataBlocks
+		case errors.Cause(errs[i]) == errDiskNotFound:
+			driveState = madmin.DriveStateOffline
+		case errors.Cause(errs[i]) == errFileNotFound, errors.Cause(errs[i]) == errVolumeNotFound:
+			fallthrough
+		case errors.Cause(dataErrs[i]) == errFileNotFound, errors.Cause(dataErrs[i]) == errVolumeNotFound:
+			driveState = madmin.DriveStateMissing
+		default:
+			// all remaining cases imply corrupt data/metadata
+			driveState = madmin.DriveStateCorrupt
 		}
-	}
+		drive := globalEndpoints.GetString(i)
+		result.DriveInfo.Before[drive] = driveState
+		// copy for 'after' state
+		result.DriveInfo.After[drive] = driveState
 
-	if numAvailableDisks == len(storageDisks) {
-		// nothing to heal in this case
-		return 0, 0, nil
+		// an online disk without valid data/metadata is
+		// outdated and can be healed.
+		if errs[i] != errDiskNotFound && v == nil {
+			outDatedDisks[i] = storageDisks[i]
+			disksToHealCount++
+		}
 	}
 
 	// If less than read quorum number of disks have all the parts
 	// of the data, we can't reconstruct the erasure-coded data.
 	if numAvailableDisks < quorum {
-		return 0, 0, toObjectErr(errXLReadQuorum, bucket, object)
+		return result, toObjectErr(errXLReadQuorum, bucket, object)
 	}
 
-	// List of disks having outdated version of the object or missing object.
-	outDatedDisks := outDatedDisks(storageDisks, availableDisks, errs, partsMetadata, bucket,
-		object)
+	if disksToHealCount == 0 {
+		// Nothing to heal!
+		return result, nil
+	}
 
-	// Number of disks that had outdated content of the given
-	// object and are online to be healed.
-	numHealedDisks := 0
-	for _, disk := range outDatedDisks {
-		if disk != nil {
-			numHealedDisks++
-		}
+	// After this point, only have to repair data on disk - so
+	// return if it is a dry-run
+	if dryRun {
+		return result, nil
 	}
 
 	// Latest xlMetaV1 for reference. If a valid metadata is not
 	// present, it is as good as object not found.
 	latestMeta, pErr := pickValidXLMeta(partsMetadata, modTime)
 	if pErr != nil {
-		return 0, 0, toObjectErr(pErr, bucket, object)
+		return result, toObjectErr(pErr, bucket, object)
 	}
 
-	for index, disk := range outDatedDisks {
+	// Clear data files of the object on outdated disks
+	for _, disk := range outDatedDisks {
 		// Before healing outdated disks, we need to remove
 		// xl.json and part files from "bucket/object/" so
 		// that rename(minioMetaBucket, "tmp/tmpuuid/",
@@ -417,18 +486,10 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i
 			continue
 		}
 
-		// errFileNotFound implies that xl.json is missing. We
-		// may have object parts still present in the object
-		// directory. This needs to be deleted for object to
-		// healed successfully.
-		if errs[index] != nil && !errors.IsErr(errs[index], errFileNotFound) {
-			continue
-		}
-
 		// List and delete the object directory, ignoring
 		// errors.
-		files, err := disk.ListDir(bucket, object)
-		if err == nil {
+		files, derr := disk.ListDir(bucket, object)
+		if derr == nil {
 			for _, entry := range files {
 				_ = disk.DeleteFile(bucket,
 					pathJoin(object, entry))
@@ -452,10 +513,10 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i
 	// Heal each part. erasureHealFile() will write the healed
 	// part to .minio/tmp/uuid/ which needs to be renamed later to
 	// the final location.
-	storage, err := NewErasureStorage(latestDisks,
-		latestMeta.Erasure.DataBlocks, latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
+	storage, err := NewErasureStorage(latestDisks, latestMeta.Erasure.DataBlocks,
+		latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
 	if err != nil {
-		return 0, 0, toObjectErr(err, bucket, object)
+		return result, toObjectErr(err, bucket, object)
 	}
 	checksums := make([][]byte, len(latestDisks))
 	for partIndex := 0; partIndex < len(latestMeta.Parts); partIndex++ {
@@ -475,7 +536,7 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i
 			erasure.BlockSize, minioMetaTmpBucket, pathJoin(tmpID, partName), partSize,
 			algorithm, checksums)
 		if hErr != nil {
-			return 0, 0, toObjectErr(hErr, bucket, object)
+			return result, toObjectErr(hErr, bucket, object)
 		}
 		// outDatedDisks that had write errors should not be
 		// written to for remaining parts, so we nil it out.
@@ -487,7 +548,7 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i
 			// a healed part checksum had a write error.
 			if file.Checksums[i] == nil {
 				outDatedDisks[i] = nil
-				numHealedDisks--
+				disksToHealCount--
 				continue
 			}
 			// append part checksums
@@ -496,8 +557,8 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i
 		}
 
 		// If all disks are having errors, we give up.
-		if numHealedDisks == 0 {
-			return 0, 0, fmt.Errorf("all disks without up-to-date data had write errors")
+		if disksToHealCount == 0 {
+			return result, fmt.Errorf("all disks without up-to-date data had write errors")
 		}
 	}
 
@@ -514,11 +575,11 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i
 	outDatedDisks, aErr = writeUniqueXLMetadata(outDatedDisks, minioMetaTmpBucket, tmpID,
 		partsMetadata, diskCount(outDatedDisks))
 	if aErr != nil {
-		return 0, 0, toObjectErr(aErr, bucket, object)
+		return result, toObjectErr(aErr, bucket, object)
 	}
 
 	// Rename from tmp location to the actual location.
-	for _, disk := range outDatedDisks {
+	for diskIndex, disk := range outDatedDisks {
 		if disk == nil {
 			continue
 		}
@@ -527,33 +588,47 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i
 		aErr = disk.RenameFile(minioMetaTmpBucket, retainSlash(tmpID), bucket,
 			retainSlash(object))
 		if aErr != nil {
-			return 0, 0, toObjectErr(errors.Trace(aErr), bucket, object)
+			return result, toObjectErr(errors.Trace(aErr), bucket, object)
 		}
+
+		realDiskIdx := unshuffleIndex(diskIndex,
+			latestMeta.Erasure.Distribution)
+		drive := globalEndpoints.GetString(realDiskIdx)
+		result.DriveInfo.After[drive] = madmin.DriveStateOk
 	}
-	return numOfflineDisks, numHealedDisks, nil
+
+	// Set the size of the object in the heal result
+	result.ObjectSize = latestMeta.Stat.Size
+
+	return result, nil
 }
 
-// HealObject heals a given object for all its missing entries.
+// HealObject - heal the given object.
+//
 // FIXME: If an object object was deleted and one disk was down,
 // and later the disk comes back up again, heal on the object
 // should delete it.
-func (xl xlObjects) HealObject(bucket, object string) (int, int, error) {
+func (xl xlObjects) HealObject(bucket, object string, dryRun bool) (
+	hr madmin.HealResultItem, err error) {
+
+	// FIXME: Metadata is read again in the healObject() call below.
 	// Read metadata files from all the disks
 	partsMetadata, errs := readAllXLMetadata(xl.storageDisks, bucket, object)
 
 	// get read quorum for this object
-	readQuorum, _, err := objectQuorumFromMeta(xl, partsMetadata, errs)
+	var readQuorum int
+	readQuorum, _, err = objectQuorumFromMeta(xl, partsMetadata, errs)
 	if err != nil {
-		return 0, 0, err
+		return hr, err
 	}
 
 	// Lock the object before healing.
 	objectLock := xl.nsMutex.NewNSLock(bucket, object)
-	if err := objectLock.GetRLock(globalHealingTimeout); err != nil {
-		return 0, 0, err
+	if lerr := objectLock.GetRLock(globalHealingTimeout); lerr != nil {
+		return hr, lerr
 	}
 	defer objectLock.RUnlock()
 
 	// Heal the object.
-	return healObject(xl.storageDisks, bucket, object, readQuorum)
+	return healObject(xl.storageDisks, bucket, object, readQuorum, dryRun)
 }