heal: Refactor heal command. (#2901)

- return errors for heal operation through rpc replies.
  - implement rotating wheel for healing status.

Fixes #2491

cmd/control-handlers.go

@@ -68,7 +68,7 @@ type HealListArgs struct {
 type HealListReply struct {
 	IsTruncated bool
 	NextMarker  string
-	Objects     []string
+	Objects     []ObjectInfo
 }
 
 // ListObjects - list all objects that needs healing.
@@ -80,35 +80,30 @@ func (c *controlAPIHandlers) ListObjectsHealHandler(args *HealListArgs, reply *H
 	if !isRPCTokenValid(args.Token) {
 		return errInvalidToken
 	}
+	if !c.IsXL {
+		return nil
+	}
 	info, err := objAPI.ListObjectsHeal(args.Bucket, args.Prefix, args.Marker, args.Delimiter, args.MaxKeys)
 	if err != nil {
 		return err
 	}
 	reply.IsTruncated = info.IsTruncated
 	reply.NextMarker = info.NextMarker
-	for _, obj := range info.Objects {
-		reply.Objects = append(reply.Objects, obj.Name)
-	}
+	reply.Objects = info.Objects
 	return nil
 }
 
-// HealObjectArgs - argument for HealObject RPC.
-type HealObjectArgs struct {
+// HealBucketArgs - arguments for HealBucket RPC.
+type HealBucketArgs struct {
 	// Authentication token generated by Login.
 	GenericArgs
 
-	// Name of the bucket.
+	// Bucket to be healed.
 	Bucket string
-
-	// Name of the object.
-	Object string
 }
 
-// HealObjectReply - reply by HealObject RPC.
-type HealObjectReply struct{}
-
-// HealObject - heal the object.
-func (c *controlAPIHandlers) HealObjectHandler(args *HealObjectArgs, reply *GenericReply) error {
+// Heals missing buckets across disks, if we have enough quorum.
+func (c *controlAPIHandlers) HealBucketHandler(args *HealBucketArgs, reply *GenericReply) error {
 	objAPI := c.ObjectAPI()
 	if objAPI == nil {
 		return errServerNotInitialized
@@ -116,15 +111,72 @@ func (c *controlAPIHandlers) HealObjectHandler(args *HealObjectArgs, reply *Gene
 	if !isRPCTokenValid(args.Token) {
 		return errInvalidToken
 	}
-	return objAPI.HealObject(args.Bucket, args.Object)
+	if !c.IsXL {
+		return nil
+	}
+	// Proceed to heal the bucket.
+	return objAPI.HealBucket(args.Bucket)
 }
 
-// HealObject - heal the object.
-func (c *controlAPIHandlers) HealDiskMetadataHandler(args *GenericArgs, reply *GenericReply) error {
+// HealObjectArgs - argument for HealObject RPC.
+type HealObjectArgs struct {
+	// Authentication token generated by Login.
+	GenericArgs
+
+	// Name of the bucket where the object
+	// needs to be healed.
+	Bucket string
+
+	// Name of the object to be healed.
+	Objects []ObjectInfo
+}
+
+// HealObjectReply - reply by HealObject RPC.
+type HealObjectReply struct {
+	Results []string
+}
+
+// HealObject heals 1000 objects at a time for missing chunks, missing metadata on a given bucket.
+func (c *controlAPIHandlers) HealObjectsHandler(args *HealObjectArgs, reply *HealObjectReply) error {
+	objAPI := c.ObjectAPI()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
 	if !isRPCTokenValid(args.Token) {
 		return errInvalidToken
 	}
-	err := repairDiskMetadata(c.StorageDisks)
+	if !c.IsXL {
+		return nil
+	}
+
+	// Heal all objects that need healing.
+	var errs = make([]error, len(args.Objects))
+	for idx, objInfo := range args.Objects {
+		errs[idx] = objAPI.HealObject(args.Bucket, objInfo.Name)
+	}
+
+	// Get all the error causes.
+	var causes = make([]string, len(args.Objects))
+	for id, err := range errs {
+		if err != nil {
+			causes[id] = err.Error()
+		}
+	}
+
+	// Save the causes.
+	reply.Results = causes
+	return nil
+}
+
+// Heals backend storage format.
+func (c *controlAPIHandlers) HealFormatHandler(args *GenericArgs, reply *GenericReply) error {
+	if !isRPCTokenValid(args.Token) {
+		return errInvalidToken
+	}
+	if !c.IsXL {
+		return nil
+	}
+	err := healFormatXL(c.StorageDisks)
 	if err != nil {
 		return err
 	}
@@ -214,6 +266,9 @@ func (c *controlAPIHandlers) TryInitHandler(args *GenericArgs, reply *GenericRep
 	if !isRPCTokenValid(args.Token) {
 		return errInvalidToken
 	}
+	if !c.IsXL {
+		return nil
+	}
 	go func() {
 		globalWakeupCh <- struct{}{}
 	}()