Add disksUnavailable healStatus const (#3990)

`disksUnavailable` healStatus constant indicates that a given object
needs healing but one or more of disks requiring heal are offline. This
can be used by admin heal API consumers to distinguish between a
successful heal and a no-op since the outdated disks were offline.

cmd/admin-handlers.go

@@ -604,6 +604,11 @@ func isDryRun(qval url.Values) bool {
 	return false
 }
 
+type healObjectResult struct {
+	HealedCount  int
+	OfflineCount int
+}
+
 // HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject&dry-run
 // - x-minio-operation = object
 // - bucket and object are both mandatory query parameters
@@ -646,14 +651,23 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	err := objLayer.HealObject(bucket, object)
+	numOfflineDisks, numHealedDisks, err := objLayer.HealObject(bucket, object)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	jsonBytes, err := json.Marshal(healObjectResult{
+		HealedCount:  numHealedDisks,
+		OfflineCount: numOfflineDisks,
+	})
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Return 200 on success.
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessResponseJSON(w, jsonBytes)
 }
 
 // HealUploadHandler - POST /?heal&bucket=mybucket&object=myobject&upload-id=myuploadID&dry-run
@@ -715,14 +729,23 @@ func (adminAPI adminAPIHandlers) HealUploadHandler(w http.ResponseWriter, r *htt
 	//object.  The 'object' corresponding to a given bucket,
 	//object and uploadID is
 	//.minio.sys/multipart/bucket/object/uploadID.
-	err := objLayer.HealObject(minioMetaMultipartBucket, uploadObj)
+	numOfflineDisks, numHealedDisks, err := objLayer.HealObject(minioMetaMultipartBucket, uploadObj)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	jsonBytes, err := json.Marshal(healObjectResult{
+		HealedCount:  numHealedDisks,
+		OfflineCount: numOfflineDisks,
+	})
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Return 200 on success.
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessResponseJSON(w, jsonBytes)
 }
 
 // HealFormatHandler - POST /?heal&dry-run

cmd/azure-unsupported.go

@@ -27,8 +27,8 @@ func (a AzureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
 }
 
 // HealObject - Not relevant.
-func (a AzureObjects) HealObject(bucket, object string) error {
-	return traceError(NotImplemented{})
+func (a AzureObjects) HealObject(bucket, object string) (int, int, error) {
+	return 0, 0, traceError(NotImplemented{})
 }
 
 // ListObjectsHeal - Not relevant.

cmd/fs-v1.go

@@ -809,8 +809,8 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs fsObjects) HealObject(bucket, object string) error {
-	return traceError(NotImplemented{})
+func (fs fsObjects) HealObject(bucket, object string) (int, int, error) {
+	return 0, 0, traceError(NotImplemented{})
 }
 
 // HealBucket - no-op for fs, Valid only for XL.

cmd/fs-v1_test.go

@@ -303,7 +303,7 @@ func TestFSHealObject(t *testing.T) {
 	defer removeAll(disk)
 
 	obj := initFSObjects(disk, t)
-	err := obj.HealObject("bucket", "object")
+	_, _, err := obj.HealObject("bucket", "object")
 	if err == nil || !isSameType(errorCause(err), NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/object-api-datatypes.go

@@ -57,6 +57,7 @@ const (
 	canHeal                             // Object can be healed
 	corrupted                           // Object can't be healed
 	quorumUnavailable                   // Object can't be healed until read quorum is available
+	canPartiallyHeal                    // Object can't be healed completely until outdated disk(s) are online.
 )
 
 // HealBucketInfo - represents healing related information of a bucket.
@@ -78,9 +79,9 @@ type BucketInfo struct {
 
 // HealObjectInfo - represents healing related information of an object.
 type HealObjectInfo struct {
-	Status              healStatus
-	MissingDataCount    int
-	MissingPartityCount int
+	Status             healStatus
+	MissingDataCount   int
+	MissingParityCount int
 }
 
 // ObjectInfo - represents object metadata.

cmd/object-api-interface.go

@@ -50,7 +50,7 @@ type ObjectLayer interface {
 	// Healing operations.
 	HealBucket(bucket string) error
 	ListBucketsHeal() (buckets []BucketInfo, err error)
-	HealObject(bucket, object string) error
+	HealObject(bucket, object string) (int, int, error)
 	ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error)
 	ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
 		delimiter string, maxUploads int) (ListMultipartsInfo, error)

cmd/xl-v1-healing-common.go

@@ -185,9 +185,9 @@ func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealObject
 	modTime, count := commonTime(listObjectModtimes(partsMetadata, errs))
 	if count < xl.readQuorum {
 		return HealObjectInfo{
-			Status:              quorumUnavailable,
-			MissingDataCount:    0,
-			MissingPartityCount: 0,
+			Status:             quorumUnavailable,
+			MissingDataCount:   0,
+			MissingParityCount: 0,
 		}
 	}
 
@@ -195,9 +195,9 @@ func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealObject
 	xlMeta, err := pickValidXLMeta(partsMetadata, modTime)
 	if err != nil {
 		return HealObjectInfo{
-			Status:              corrupted,
-			MissingDataCount:    0,
-			MissingPartityCount: 0,
+			Status:             corrupted,
+			MissingDataCount:   0,
+			MissingParityCount: 0,
 		}
 	}
 
@@ -206,11 +206,16 @@ func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealObject
 	missingDataCount := 0
 	missingParityCount := 0
 
+	disksMissing := false
 	for i, err := range errs {
 		// xl.json is not found, which implies the erasure
 		// coded blocks are unavailable in the corresponding disk.
 		// First half of the disks are data and the rest are parity.
-		if realErr := errorCause(err); realErr == errFileNotFound || realErr == errDiskNotFound {
+		switch realErr := errorCause(err); realErr {
+		case errDiskNotFound:
+			disksMissing = true
+			fallthrough
+		case errFileNotFound:
 			if xlMeta.Erasure.Distribution[i]-1 < xl.dataBlocks {
 				missingDataCount++
 			} else {
@@ -219,12 +224,22 @@ func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealObject
 		}
 	}
 
+	// The object may not be healed completely, since some of the
+	// disks needing healing are unavailable.
+	if disksMissing {
+		return HealObjectInfo{
+			Status:             canPartiallyHeal,
+			MissingDataCount:   missingDataCount,
+			MissingParityCount: missingParityCount,
+		}
+	}
+
 	// This object can be healed. We have enough object metadata
 	// to reconstruct missing erasure coded blocks.
 	return HealObjectInfo{
-		Status:              canHeal,
-		MissingDataCount:    missingDataCount,
-		MissingPartityCount: missingParityCount,
+		Status:             canHeal,
+		MissingDataCount:   missingDataCount,
+		MissingParityCount: missingParityCount,
 	}
 }
 

cmd/xl-v1-healing.go

@@ -125,7 +125,7 @@ func healBucketMetadata(storageDisks []StorageAPI, bucket string, readQuorum int
 		metaLock.RLock()
 		defer metaLock.RUnlock()
 		// Heals the given file at metaPath.
-		if err := healObject(storageDisks, minioMetaBucket, metaPath, readQuorum); err != nil && !isErrObjectNotFound(err) {
+		if _, _, err := healObject(storageDisks, minioMetaBucket, metaPath, readQuorum); err != nil && !isErrObjectNotFound(err) {
 			return err
 		} // Success.
 		return nil
@@ -313,18 +313,18 @@ func quickHeal(storageDisks []StorageAPI, writeQuorum int, readQuorum int) error
 }
 
 // Heals an object only the corrupted/missing erasure blocks.
-func healObject(storageDisks []StorageAPI, bucket string, object string, quorum int) error {
+func healObject(storageDisks []StorageAPI, bucket string, object string, quorum int) (int, int, error) {
 	partsMetadata, errs := readAllXLMetadata(storageDisks, bucket, object)
 	// readQuorum suffices for xl.json since we use monotonic
 	// system time to break the tie when a split-brain situation
 	// arises.
 	if reducedErr := reduceReadQuorumErrs(errs, nil, quorum); reducedErr != nil {
-		return toObjectErr(reducedErr, bucket, object)
+		return 0, 0, toObjectErr(reducedErr, bucket, object)
 	}
 
 	if !xlShouldHeal(storageDisks, partsMetadata, errs, bucket, object) {
 		// There is nothing to heal.
-		return nil
+		return 0, 0, nil
 	}
 
 	// List of disks having latest version of the object.
@@ -333,12 +333,16 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, quorum
 	// List of disks having all parts as per latest xl.json.
 	availableDisks, errs, aErr := disksWithAllParts(latestDisks, partsMetadata, errs, bucket, object)
 	if aErr != nil {
-		return toObjectErr(aErr, bucket, object)
+		return 0, 0, toObjectErr(aErr, bucket, object)
 	}
 
 	numAvailableDisks := 0
-	for _, disk := range availableDisks {
-		if disk != nil {
+	numOfflineDisks := 0
+	for index, disk := range availableDisks {
+		switch {
+		case disk == nil, errs[index] == errDiskNotFound:
+			numOfflineDisks++
+		case disk != nil:
 			numAvailableDisks++
 		}
 	}
@@ -346,18 +350,25 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, quorum
 	// If less than read quorum number of disks have all the parts
 	// of the data, we can't reconstruct the erasure-coded data.
 	if numAvailableDisks < quorum {
-		return toObjectErr(errXLReadQuorum, bucket, object)
+		return 0, 0, toObjectErr(errXLReadQuorum, bucket, object)
 	}
 
 	// List of disks having outdated version of the object or missing object.
 	outDatedDisks := outDatedDisks(storageDisks, availableDisks, errs, partsMetadata,
 		bucket, object)
 
+	numHealedDisks := 0
+	for _, disk := range outDatedDisks {
+		if disk != nil {
+			numHealedDisks++
+		}
+	}
+
 	// Latest xlMetaV1 for reference. If a valid metadata is not
 	// present, it is as good as object not found.
 	latestMeta, pErr := pickValidXLMeta(partsMetadata, modTime)
 	if pErr != nil {
-		return toObjectErr(pErr, bucket, object)
+		return 0, 0, toObjectErr(pErr, bucket, object)
 	}
 
 	for index, disk := range outDatedDisks {
@@ -389,14 +400,14 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, quorum
 		for _, part := range outDatedMeta.Parts {
 			dErr := disk.DeleteFile(bucket, pathJoin(object, part.Name))
 			if dErr != nil && !isErr(dErr, errFileNotFound) {
-				return toObjectErr(traceError(dErr), bucket, object)
+				return 0, 0, toObjectErr(traceError(dErr), bucket, object)
 			}
 		}
 
 		// Delete xl.json file. Ignore if xl.json not found.
 		dErr := disk.DeleteFile(bucket, pathJoin(object, xlMetaJSONFile))
 		if dErr != nil && !isErr(dErr, errFileNotFound) {
-			return toObjectErr(traceError(dErr), bucket, object)
+			return 0, 0, toObjectErr(traceError(dErr), bucket, object)
 		}
 	}
 
@@ -425,7 +436,7 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, quorum
 			minioMetaTmpBucket, pathJoin(tmpID, partName),
 			partSize, erasure.BlockSize, erasure.DataBlocks, erasure.ParityBlocks, sumInfo.Algorithm)
 		if hErr != nil {
-			return toObjectErr(hErr, bucket, object)
+			return 0, 0, toObjectErr(hErr, bucket, object)
 		}
 		for index, sum := range checkSums {
 			if outDatedDisks[index] != nil {
@@ -450,7 +461,7 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, quorum
 	// Generate and write `xl.json` generated from other disks.
 	aErr = writeUniqueXLMetadata(outDatedDisks, minioMetaTmpBucket, tmpID, partsMetadata, diskCount(outDatedDisks))
 	if aErr != nil {
-		return toObjectErr(aErr, bucket, object)
+		return 0, 0, toObjectErr(aErr, bucket, object)
 	}
 
 	// Rename from tmp location to the actual location.
@@ -461,22 +472,22 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, quorum
 		// Remove any lingering partial data from current namespace.
 		aErr = disk.DeleteFile(bucket, retainSlash(object))
 		if aErr != nil && aErr != errFileNotFound {
-			return toObjectErr(traceError(aErr), bucket, object)
+			return 0, 0, toObjectErr(traceError(aErr), bucket, object)
 		}
 		// Attempt a rename now from healed data to final location.
 		aErr = disk.RenameFile(minioMetaTmpBucket, retainSlash(tmpID), bucket, retainSlash(object))
 		if aErr != nil {
-			return toObjectErr(traceError(aErr), bucket, object)
+			return 0, 0, toObjectErr(traceError(aErr), bucket, object)
 		}
 	}
-	return nil
+	return numOfflineDisks, numHealedDisks, nil
 }
 
 // HealObject heals a given object for all its missing entries.
 // FIXME: If an object object was deleted and one disk was down,
 // and later the disk comes back up again, heal on the object
 // should delete it.
-func (xl xlObjects) HealObject(bucket, object string) error {
+func (xl xlObjects) HealObject(bucket, object string) (int, int, error) {
 	// Lock the object before healing.
 	objectLock := globalNSMutex.NewNSLock(bucket, object)
 	objectLock.RLock()

cmd/xl-v1-healing_test.go

@@ -558,7 +558,7 @@ func TestHealObjectXL(t *testing.T) {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
 
-	err = obj.HealObject(bucket, object)
+	_, _, err = obj.HealObject(bucket, object)
 	if err != nil {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
@@ -574,7 +574,7 @@ func TestHealObjectXL(t *testing.T) {
 	}
 
 	// Try healing now, expect to receive errDiskNotFound.
-	err = obj.HealObject(bucket, object)
+	_, _, err = obj.HealObject(bucket, object)
 	if errorCause(err) != errDiskNotFound {
 		t.Errorf("Expected %v but received %v", errDiskNotFound, err)
 	}

cmd/xl-v1-object_test.go

@@ -312,7 +312,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = xl.HealObject(bucket, object)
+	_, _, err = xl.HealObject(bucket, object)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -336,7 +336,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = xl.HealObject(bucket, object)
+	_, _, err = xl.HealObject(bucket, object)
 	if err != nil {
 		t.Fatal(err)
 	}