Implement heal-upload admin API (#3914)

This API is meant for administrative tools like mc-admin to heal an
ongoing multipart upload on a Minio server.  N B This set of admin
APIs apply only for Minio servers.

`github.com/minio/minio/pkg/madmin` provides a go SDK for this (and
other admin) operations.  Specifically,

  func HealUpload(bucket, object, uploadID string, dryRun bool) error

Sample admin API request:
POST
/?heal&bucket=mybucket&object=myobject&upload-id=myuploadID&dry-run
- Header(s): ["x-minio-operation"] = "upload"

Notes:
- bucket, object and upload-id are mandatory query parameters
- if dry-run is set, API returns success if all parameters passed are
  valid.

cmd/admin-handlers.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"path"
 	"strconv"
 	"time"
 )
@@ -49,6 +50,7 @@ const (
 	mgmtDryRun         mgmtQueryKey = "dry-run"
 	mgmtUploadIDMarker mgmtQueryKey = "upload-id-marker"
 	mgmtMaxUploads     mgmtQueryKey = "max-uploads"
+	mgmtUploadID       mgmtQueryKey = "upload-id"
 )
 
 // ServerVersion - server version
@@ -654,6 +656,75 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
 	writeSuccessResponseHeadersOnly(w)
 }
 
+// HealUploadHandler - POST /?heal&bucket=mybucket&object=myobject&upload-id=myuploadID&dry-run
+// - x-minio-operation = upload
+// - bucket, object and upload-id are mandatory query parameters
+// Heal a given upload, if present.
+func (adminAPI adminAPIHandlers) HealUploadHandler(w http.ResponseWriter, r *http.Request) {
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, adminAPIErr, r.URL)
+		return
+	}
+
+	vars := r.URL.Query()
+	bucket := vars.Get(string(mgmtBucket))
+	object := vars.Get(string(mgmtObject))
+	uploadID := vars.Get(string(mgmtUploadID))
+	uploadObj := path.Join(bucket, object, uploadID)
+
+	// Validate bucket and object names as supplied via query
+	// parameters.
+	if err := checkBucketAndObjectNames(bucket, object); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Validate the bucket and object w.r.t backend representation
+	// of an upload.
+	if err := checkBucketAndObjectNames(minioMetaMultipartBucket,
+		uploadObj); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Check if upload exists.
+	if _, err := objLayer.GetObjectInfo(minioMetaMultipartBucket,
+		uploadObj); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// if dry-run is set in query params then perform validations
+	// and return success.
+	if isDryRun(vars) {
+		writeSuccessResponseHeadersOnly(w)
+		return
+	}
+
+	//We are able to use HealObject for healing an upload since an
+	//ongoing upload has the same backend representation as an
+	//object.  The 'object' corresponding to a given bucket,
+	//object and uploadID is
+	//.minio.sys/multipart/bucket/object/uploadID.
+	err := objLayer.HealObject(minioMetaMultipartBucket, uploadObj)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Return 200 on success.
+	writeSuccessResponseHeadersOnly(w)
+}
+
 // HealFormatHandler - POST /?heal&dry-run
 // - x-minio-operation = format
 // - bucket and object are both mandatory query parameters

cmd/admin-handlers_test.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -1048,6 +1049,160 @@ func TestHealObjectHandler(t *testing.T) {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
 		}
 	}
+
+}
+
+// buildAdminRequest - helper function to build an admin API request.
+func buildAdminRequest(queryVal url.Values, opHdr, method string,
+	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
+	req, err := newTestRequest(method, "/?"+queryVal.Encode(), contentLength, bodySeeker)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	req.Header.Set(minioAdminOpHeader, opHdr)
+
+	cred := serverConfig.GetCredential()
+	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	return req, nil
+}
+
+// mkHealUploadQuery - helper to build HealUploadHandler query string.
+func mkHealUploadQuery(bucket, object, uploadID, dryRun string) url.Values {
+	queryVal := url.Values{}
+	queryVal.Set(string(mgmtBucket), bucket)
+	queryVal.Set(string(mgmtObject), object)
+	queryVal.Set(string(mgmtUploadID), uploadID)
+	queryVal.Set("heal", "")
+	queryVal.Set(string(mgmtDryRun), dryRun)
+	return queryVal
+}
+
+// TestHealUploadHandler - test for HealUploadHandler.
+func TestHealUploadHandler(t *testing.T) {
+	adminTestBed, err := prepareAdminXLTestBed()
+	if err != nil {
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+	}
+	defer adminTestBed.TearDown()
+
+	// Create an object myobject under bucket mybucket.
+	bucketName := "mybucket"
+	objName := "myobject"
+	err = adminTestBed.objLayer.MakeBucket(bucketName)
+	if err != nil {
+		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
+	}
+
+	// Create a new multipart upload.
+	uploadID, err := adminTestBed.objLayer.NewMultipartUpload(bucketName, objName, nil)
+	if err != nil {
+		t.Fatalf("Failed to create a new multipart upload %s/%s - %v",
+			bucketName, objName, err)
+	}
+
+	// Upload a part.
+	partID := 1
+	_, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID,
+		partID, int64(len("hello")), bytes.NewReader([]byte("hello")), "", "")
+	if err != nil {
+		t.Fatalf("Failed to upload part %d of %s/%s - %v", partID,
+			bucketName, objName, err)
+	}
+
+	testCases := []struct {
+		bucket     string
+		object     string
+		dryrun     string
+		statusCode int
+	}{
+		// 1. Valid test case.
+		{
+			bucket:     bucketName,
+			object:     objName,
+			statusCode: http.StatusOK,
+		},
+		// 2. Invalid bucket name.
+		{
+			bucket:     `invalid\\Bucket`,
+			object:     "myobject",
+			statusCode: http.StatusBadRequest,
+		},
+		// 3. Bucket not found.
+		{
+			bucket:     "bucketnotfound",
+			object:     "myobject",
+			statusCode: http.StatusNotFound,
+		},
+		// 4. Invalid object name.
+		{
+			bucket:     bucketName,
+			object:     `invalid\\Object`,
+			statusCode: http.StatusBadRequest,
+		},
+		// 5. Object not found.
+		{
+			bucket:     bucketName,
+			object:     "objectnotfound",
+			statusCode: http.StatusNotFound,
+		},
+		// 6. Valid test case with dry-run.
+		{
+			bucket:     bucketName,
+			object:     objName,
+			dryrun:     "yes",
+			statusCode: http.StatusOK,
+		},
+	}
+	for i, test := range testCases {
+		// Prepare query params.
+		queryVal := mkHealUploadQuery(test.bucket, test.object, uploadID, test.dryrun)
+		req, err := buildAdminRequest(queryVal, "upload", http.MethodPost, 0, nil)
+		if err != nil {
+			t.Fatalf("Test %d - Failed to construct heal object request - %v", i+1, err)
+		}
+		rec := httptest.NewRecorder()
+		adminTestBed.mux.ServeHTTP(rec, req)
+		if test.statusCode != rec.Code {
+			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
+		}
+	}
+
+	sample := testCases[0]
+	// Modify authorization header after signing to test signature
+	// mismatch handling.
+	queryVal := mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
+	req, err := buildAdminRequest(queryVal, "upload", "POST", 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct heal object request - %v", err)
+	}
+
+	// Set x-amz-date to a date different than time of signing.
+	req.Header.Set("x-amz-date", time.Time{}.Format(iso8601Format))
+
+	rec := httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusForbidden {
+		t.Errorf("Expected %d but received %d", http.StatusBadRequest, rec.Code)
+	}
+
+	// Set objectAPI to nil to test Server not initialized case.
+	resetGlobalObjectAPI()
+	queryVal = mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
+	req, err = buildAdminRequest(queryVal, "upload", "POST", 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct heal object request - %v", err)
+	}
+
+	rec = httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Errorf("Expected %d but received %d", http.StatusServiceUnavailable, rec.Code)
+	}
 }
 
 // TestHealFormatHandler - test for HealFormatHandler.
@@ -1061,21 +1216,11 @@ func TestHealFormatHandler(t *testing.T) {
 	// Prepare query params for heal-format mgmt REST API.
 	queryVal := url.Values{}
 	queryVal.Set("heal", "")
-	req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
+	req, err := buildAdminRequest(queryVal, "format", "POST", 0, nil)
 	if err != nil {
 		t.Fatalf("Failed to construct heal object request - %v", err)
 	}
 
-	// Set x-minio-operation header to format.
-	req.Header.Set(minioAdminOpHeader, "format")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1105,21 +1250,11 @@ func TestGetConfigHandler(t *testing.T) {
 	queryVal := url.Values{}
 	queryVal.Set("config", "")
 
-	req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil)
+	req, err := buildAdminRequest(queryVal, "get", http.MethodGet, 0, nil)
 	if err != nil {
 		t.Fatalf("Failed to construct get-config object request - %v", err)
 	}
 
-	// Set x-minio-operation header to get.
-	req.Header.Set(minioAdminOpHeader, "get")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1154,21 +1289,12 @@ func TestSetConfigHandler(t *testing.T) {
 	queryVal := url.Values{}
 	queryVal.Set("config", "")
 
-	req, err := newTestRequest("PUT", "/?"+queryVal.Encode(), int64(len(configJSON)), bytes.NewReader(configJSON))
+	req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(configJSON)),
+		bytes.NewReader(configJSON))
 	if err != nil {
 		t.Fatalf("Failed to construct get-config object request - %v", err)
 	}
 
-	// Set x-minio-operation header to set.
-	req.Header.Set(minioAdminOpHeader, "set")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1288,9 +1414,9 @@ func TestWriteSetConfigResponse(t *testing.T) {
 	}
 }
 
-// mkUploadsHealQuery - helper function to construct query values for
+// mkListUploadsHealQuery - helper function to construct query values for
 // listUploadsHeal.
-func mkUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
+func mkListUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
 
 	queryVal := make(url.Values)
 	queryVal.Set("heal", "")
@@ -1401,19 +1527,13 @@ func TestListHealUploadsHandler(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		queryVal := mkUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
+		queryVal := mkListUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
 
-		req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil)
+		req, err := buildAdminRequest(queryVal, "list-uploads", http.MethodGet, 0, nil)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to construct list uploads needing heal request - %v", i+1, err)
 		}
-		req.Header.Set(minioAdminOpHeader, "list-uploads")
 
-		cred := serverConfig.GetCredential()
-		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to sign list uploads needing heal request - %v", i+1, err)
-		}
 		rec := httptest.NewRecorder()
 		adminTestBed.mux.ServeHTTP(rec, req)
 		if test.statusCode != rec.Code {

cmd/admin-router.go

@@ -64,6 +64,8 @@ func registerAdminRouter(mux *router.Router) {
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
 	// Heal Format.
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
+	// Heal Uploads.
+	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "upload").HandlerFunc(adminAPI.HealUploadHandler)
 
 	/// Config operations
 

cmd/xl-v1-healing.go

@@ -477,10 +477,6 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, quorum
 // and later the disk comes back up again, heal on the object
 // should delete it.
 func (xl xlObjects) HealObject(bucket, object string) error {
-	if err := checkGetObjArgs(bucket, object); err != nil {
-		return err
-	}
-
 	// Lock the object before healing.
 	objectLock := globalNSMutex.NewNSLock(bucket, object)
 	objectLock.RLock()

cmd/xl-v1-healing_test.go

@@ -17,7 +17,9 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
+	"path/filepath"
 	"testing"
 )
 
@@ -486,3 +488,73 @@ func TestListBucketsHeal(t *testing.T) {
 		t.Fatalf("Name of missing bucket is incorrect, expected: %s, found: %s", corruptedBucketName, buckets[0].Name)
 	}
 }
+
+// Tests healing of object.
+func TestHealObjectXL(t *testing.T) {
+	root, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeAll(root)
+
+	nDisks := 16
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	endpoints, err := parseStorageEndpoints(fsDirs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Everything is fine, should return nil
+	obj, _, err := initObjectLayer(endpoints)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := "bucket"
+	object := "object"
+	data := []byte("hello")
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	_, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil, "")
+	if err != nil {
+		t.Fatalf("Failed to put an object - %v", err)
+	}
+
+	// Remove the object backend files from the first disk.
+	xl := obj.(*xlObjects)
+	firstDisk := xl.storageDisks[0]
+	err = firstDisk.DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile))
+	if err != nil {
+		t.Fatalf("Failed to delete a file - %v", err)
+	}
+
+	err = obj.HealObject(bucket, object)
+	if err != nil {
+		t.Fatalf("Failed to heal object - %v", err)
+	}
+
+	_, err = firstDisk.StatFile(bucket, filepath.Join(object, xlMetaJSONFile))
+	if err != nil {
+		t.Errorf("Expected xl.json file to be present but stat failed - %v", err)
+	}
+
+	// Nil more than half the disks, to remove write quorum.
+	for i := 0; i <= len(xl.storageDisks)/2; i++ {
+		xl.storageDisks[i] = nil
+	}
+
+	// Try healing now, expect to receive errDiskNotFound.
+	err = obj.HealObject(bucket, object)
+	if errorCause(err) != errDiskNotFound {
+		t.Errorf("Expected %v but received %v", errDiskNotFound, err)
+	}
+}