Add disksUnavailable healStatus const (#3990)

`disksUnavailable` healStatus constant indicates that a given object
needs healing but one or more of disks requiring heal are offline. This
can be used by admin heal API consumers to distinguish between a
successful heal and a no-op since the outdated disks were offline.

pkg/madmin/API.md

@@ -242,18 +242,19 @@ __Example__
 ```
 
 <a name="HealObject"></a>
-### HealObject(bucket, object string, isDryRun bool) error
+### HealObject(bucket, object string, isDryRun bool) (HealObjectResult, error)
 If object is successfully healed returns nil, otherwise returns error indicating the reason for failure. If isDryRun is true, then the object is not healed, but heal object request is validated by the server. e.g, if the object exists, if object name is valid etc.
 
 __Example__
 
 ``` go
-    isDryRun := false
-    err := madmClnt.HealObject("mybucket", "myobject", isDryRun)
+    isDryRun = false
+    healResult, err := madmClnt.HealObject("mybucket", "myobject", isDryRun)
     if err != nil {
         log.Fatalln(err)
     }
-    log.Println("successfully healed mybucket/myobject")
+
+    log.Println("Heal-object result: ", healResult)
 
 ```
 
@@ -323,17 +324,17 @@ __Example__
 ```
 
 <a name="HealUpload"></a>
-### HealUpload(bucket, object, uploadID string, isDryRun bool) error
+### HealUpload(bucket, object, uploadID string, isDryRun bool) (HealObjectResult, error)
 If upload is successfully healed returns nil, otherwise returns error indicating the reason for failure. If isDryRun is true, then the upload is not healed, but heal upload request is validated by the server. e.g, if the upload exists, if upload name is valid etc.
 
 ``` go
     isDryRun = false
-    err = madmClnt.HealUpload("mybucket", "myobject", "myuploadID", isDryRun)
+    healResult, err := madmClnt.HealUpload("mybucket", "myobject", "myUploadID", isDryRun)
     if err != nil {
         log.Fatalln(err)
     }
 
-    log.Println("successfully healed mybucket/myobject/myuploadID")
+    log.Println("Heal-upload result: ", healResult)
 ```
 
 ## 5. Config operations

pkg/madmin/examples/heal-object.go

@@ -41,17 +41,17 @@ func main() {
 
 	// Heal object mybucket/myobject - dry run.
 	isDryRun := true
-	err = madmClnt.HealObject("mybucket", "myobject", isDryRun)
+	_, err = madmClnt.HealObject("mybucket", "myobject", isDryRun)
 	if err != nil {
 		log.Fatalln(err)
 	}
 
 	// Heal object mybucket/myobject - this time for real.
 	isDryRun = false
-	err = madmClnt.HealObject("mybucket", "myobject", isDryRun)
+	healResult, err := madmClnt.HealObject("mybucket", "myobject", isDryRun)
 	if err != nil {
 		log.Fatalln(err)
 	}
 
-	log.Println("successfully healed mybucket/myobject")
+	log.Printf("heal result: %#v\n", healResult)
 }

pkg/madmin/examples/heal-objects-list.go

@@ -65,6 +65,8 @@ func main() {
 			switch healInfo := *object.HealObjectInfo; healInfo.Status {
 			case madmin.CanHeal:
 				fmt.Println(object.Key, " can be healed.")
+			case madmin.CanPartiallyHeal:
+				fmt.Println(object.Key, " can't be healed completely, some disks are offline.")
 			case madmin.QuorumUnavailable:
 				fmt.Println(object.Key, " can't be healed until quorum is available.")
 			case madmin.Corrupted:

pkg/madmin/examples/heal-upload.go

@@ -41,17 +41,17 @@ func main() {
 
 	// Heal upload mybucket/myobject/uploadID - dry run.
 	isDryRun := true
-	err = madmClnt.HealUpload("mybucket", "myobject", "myuploadID", isDryRun)
+	_, err = madmClnt.HealUpload("mybucket", "myobject", "myUploadID", isDryRun)
 	if err != nil {
 		log.Fatalln(err)
 	}
 
 	// Heal upload mybucket/myobject/uploadID - this time for real.
 	isDryRun = false
-	err = madmClnt.HealUpload("mybucket", "myobject", "myuploadID", isDryRun)
+	healResult, err := madmClnt.HealUpload("mybucket", "myobject", "myUploadID", isDryRun)
 	if err != nil {
 		log.Fatalln(err)
 	}
 
-	log.Println("successfully healed mybucket/myobject/myuploadID")
+	log.Printf("Heal result for mybucket/myobject/myUploadID: %#v\n", healResult)
 }

pkg/madmin/examples/heal-uploads-list.go

@@ -64,6 +64,8 @@ func main() {
 			switch healInfo := *upload.HealUploadInfo; healInfo.Status {
 			case madmin.CanHeal:
 				fmt.Println(upload.Key, " can be healed.")
+			case madmin.CanPartiallyHeal:
+				fmt.Println(upload.Key, " can be healed partially. Some disks may be offline.")
 			case madmin.QuorumUnavailable:
 				fmt.Println(upload.Key, " can't be healed until quorum is available.")
 			case madmin.Corrupted:

pkg/madmin/heal-commands.go

@@ -20,8 +20,10 @@
 package madmin
 
 import (
+	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"time"
@@ -104,8 +106,12 @@ const (
 	CanHeal
 	// Corrupted - Object can't be healed
 	Corrupted
-	// QuorumUnavailable - Object can't be healed until read quorum is available
+	// QuorumUnavailable - Object can't be healed until read
+	// quorum is available
 	QuorumUnavailable
+	// CanPartiallyHeal - Object can't be healed completely until
+	// disks with missing parts come online
+	CanPartiallyHeal
 )
 
 // HealBucketInfo - represents healing related information of a bucket.
@@ -127,9 +133,9 @@ type BucketInfo struct {
 
 // HealObjectInfo - represents healing related information of an object.
 type HealObjectInfo struct {
-	Status              HealStatus
-	MissingDataCount    int
-	MissingPartityCount int
+	Status             HealStatus
+	MissingDataCount   int
+	MissingParityCount int
 }
 
 // ObjectInfo container for object metadata.
@@ -434,7 +440,7 @@ func (adm *AdminClient) HealBucket(bucket string, dryrun bool) error {
 }
 
 // HealUpload - Heal the given upload.
-func (adm *AdminClient) HealUpload(bucket, object, uploadID string, dryrun bool) error {
+func (adm *AdminClient) HealUpload(bucket, object, uploadID string, dryrun bool) (HealObjectResult, error) {
 	// Construct query params.
 	queryVal := url.Values{}
 	queryVal.Set("heal", "")
@@ -460,18 +466,40 @@ func (adm *AdminClient) HealUpload(bucket, object, uploadID string, dryrun bool)
 
 	defer closeResponse(resp)
 	if err != nil {
-		return err
+		return HealObjectResult{}, err
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return httpRespToErrorResponse(resp)
+		return HealObjectResult{}, httpRespToErrorResponse(resp)
 	}
 
-	return nil
+	// Healing is not performed so heal object result is empty.
+	if dryrun {
+		return HealObjectResult{}, nil
+	}
+
+	jsonBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return HealObjectResult{}, err
+	}
+
+	healResult := HealObjectResult{}
+	err = json.Unmarshal(jsonBytes, &healResult)
+	if err != nil {
+		return HealObjectResult{}, err
+	}
+
+	return healResult, nil
+}
+
+// HealObjectResult - represents result of heal-object admin API.
+type HealObjectResult struct {
+	HealedCount  int // number of disks that were healed.
+	OfflineCount int // number of disks that needed healing but were offline.
 }
 
 // HealObject - Heal the given object.
-func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) error {
+func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) (HealObjectResult, error) {
 	// Construct query params.
 	queryVal := url.Values{}
 	queryVal.Set("heal", "")
@@ -494,14 +522,30 @@ func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) error {
 
 	defer closeResponse(resp)
 	if err != nil {
-		return err
+		return HealObjectResult{}, err
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return httpRespToErrorResponse(resp)
+		return HealObjectResult{}, httpRespToErrorResponse(resp)
 	}
 
-	return nil
+	// Healing is not performed so heal object result is empty.
+	if dryrun {
+		return HealObjectResult{}, nil
+	}
+
+	jsonBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return HealObjectResult{}, err
+	}
+
+	healResult := HealObjectResult{}
+	err = json.Unmarshal(jsonBytes, &healResult)
+	if err != nil {
+		return HealObjectResult{}, err
+	}
+
+	return healResult, nil
 }
 
 // HealFormat - heal storage format on available disks.