mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Implement mgmt REST APIs to heal storage format. (#3604)
* Implement heal format REST API handler * Implement admin peer rpc handler to re-initialize storage * Implement HealFormat API in pkg/madmin * Update pkg/madmin API.md to incl. HealFormat * Added unit tests for ReInitDisks rpc handler and HealFormatHandler
This commit is contained in:
parent
4e926b292f
commit
586058f079
@ -19,6 +19,7 @@ package cmd
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@ -374,6 +375,7 @@ func (adminAPI adminAPIHandlers) ListBucketsHealHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
// HealBucketHandler - POST /?heal&bucket=mybucket
|
||||
// - x-minio-operation = bucket
|
||||
// - bucket is mandatory query parameter
|
||||
// Heal a given bucket, if present.
|
||||
func (adminAPI adminAPIHandlers) HealBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@ -425,6 +427,7 @@ func isDryRun(qval url.Values) bool {
|
||||
}
|
||||
|
||||
// HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject
|
||||
// - x-minio-operation = object
|
||||
// - bucket and object are both mandatory query parameters
|
||||
// Heal a given object, if present.
|
||||
func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@ -473,3 +476,69 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
|
||||
// Return 200 on success.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// HealFormatHandler - POST /?heal
|
||||
// - x-minio-operation = format
|
||||
// - bucket and object are both mandatory query parameters
|
||||
// Heal a given object, if present.
|
||||
func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkRequestAuthType(r, "", "", "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponse(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if this setup is an erasure code backend, since
|
||||
// heal-format is only applicable to single node XL and
|
||||
// distributed XL setup.
|
||||
if !globalIsXL {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a new set of storage instances to heal format.json.
|
||||
bootstrapDisks, err := initStorageDisks(globalEndpoints)
|
||||
if err != nil {
|
||||
fmt.Println(traceError(err))
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Heal format.json on available storage.
|
||||
err = healFormatXL(bootstrapDisks)
|
||||
if err != nil {
|
||||
fmt.Println(traceError(err))
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Instantiate new object layer with newly formatted storage.
|
||||
newObjectAPI, err := newXLObjects(bootstrapDisks)
|
||||
if err != nil {
|
||||
fmt.Println(traceError(err))
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Set object layer with newly formatted storage to globalObjectAPI.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = newObjectAPI
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Shutdown storage belonging to old object layer instance.
|
||||
objectAPI.Shutdown()
|
||||
|
||||
// Inform peers to reinitialize storage with newly formatted storage.
|
||||
reInitPeerDisks(globalAdminPeers)
|
||||
|
||||
// Return 200 on success.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
@ -29,6 +29,79 @@ import (
|
||||
router "github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// adminXLTestBed - encapsulates subsystems that need to be setup for
|
||||
// admin-handler unit tests.
|
||||
type adminXLTestBed struct {
|
||||
configPath string
|
||||
xlDirs []string
|
||||
objLayer ObjectLayer
|
||||
mux *router.Router
|
||||
}
|
||||
|
||||
// prepareAdminXLTestBed - helper function that setups a single-node
|
||||
// XL backend for admin-handler tests.
|
||||
func prepareAdminXLTestBed() (*adminXLTestBed, error) {
|
||||
// reset global variables to start afresh.
|
||||
resetTestGlobals()
|
||||
// Initialize minio server config.
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Initializing objectLayer for HealFormatHandler.
|
||||
objLayer, xlDirs, xlErr := initTestXLObjLayer()
|
||||
if xlErr != nil {
|
||||
return nil, xlErr
|
||||
}
|
||||
|
||||
// Set globalEndpoints for a single node XL setup.
|
||||
for _, xlDir := range xlDirs {
|
||||
globalEndpoints = append(globalEndpoints, &url.URL{
|
||||
Path: xlDir,
|
||||
})
|
||||
}
|
||||
|
||||
// Set globalIsXL to indicate that the setup uses an erasure code backend.
|
||||
globalIsXL = true
|
||||
|
||||
// initialize NSLock.
|
||||
isDistXL := false
|
||||
initNSLock(isDistXL)
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := router.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
|
||||
return &adminXLTestBed{
|
||||
configPath: rootPath,
|
||||
xlDirs: xlDirs,
|
||||
objLayer: objLayer,
|
||||
mux: adminRouter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TearDown - method that resets the test bed for subsequent unit
|
||||
// tests to start afresh.
|
||||
func (atb *adminXLTestBed) TearDown() {
|
||||
removeAll(atb.configPath)
|
||||
removeRoots(atb.xlDirs)
|
||||
resetTestGlobals()
|
||||
}
|
||||
|
||||
// initTestObjLayer - Helper function to initialize an XL-based object
|
||||
// layer and set globalObjectAPI.
|
||||
func initTestXLObjLayer() (ObjectLayer, []string, error) {
|
||||
objLayer, xlDirs, xlErr := prepareXL()
|
||||
if xlErr != nil {
|
||||
return nil, nil, xlErr
|
||||
}
|
||||
// Make objLayer available to all internal services via globalObjectAPI.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
return objLayer, xlDirs, nil
|
||||
}
|
||||
|
||||
// cmdType - Represents different service subcomands like status, stop
|
||||
// and restart.
|
||||
type cmdType int
|
||||
@ -115,17 +188,11 @@ func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Requ
|
||||
// testServicesCmdHandler - parametrizes service subcommand tests on
|
||||
// cmdType value.
|
||||
func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing.T) {
|
||||
// reset globals.
|
||||
// this is to make sure that the tests are not affected by modified value.
|
||||
resetTestGlobals()
|
||||
// initialize NSLock.
|
||||
initNSLock(false)
|
||||
// Initialize configuration for access/secret credentials.
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize server config. %s", err)
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls. Note: In a
|
||||
// single node setup, this degenerates to a simple function
|
||||
@ -139,29 +206,12 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
|
||||
if cmd == statusCmd {
|
||||
// Initializing objectLayer and corresponding
|
||||
// []StorageAPI since DiskInfo() method requires it.
|
||||
objLayer, xlDirs, xlErr := prepareXL()
|
||||
if xlErr != nil {
|
||||
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
|
||||
}
|
||||
defer removeRoots(xlDirs)
|
||||
// Make objLayer available to all internal services via globalObjectAPI.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
}
|
||||
|
||||
// Setting up a go routine to simulate ServerMux's
|
||||
// handleServiceSignals for stop and restart commands.
|
||||
if cmd == restartCmd {
|
||||
go testServiceSignalReceiver(cmd, t)
|
||||
}
|
||||
credentials := serverConfig.GetCredential()
|
||||
adminRouter := router.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
|
||||
var body []byte
|
||||
|
||||
if cmd == setCreds {
|
||||
@ -174,7 +224,7 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminRouter.ServeHTTP(rec, req)
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
|
||||
if cmd == statusCmd {
|
||||
expectedInfo := newObjectLayerFn().StorageInfo()
|
||||
@ -232,17 +282,11 @@ func mkLockQueryVal(bucket, prefix, relTimeStr string) url.Values {
|
||||
|
||||
// Test for locks list management REST API.
|
||||
func TestListLocksHandler(t *testing.T) {
|
||||
// reset globals.
|
||||
// this is to make sure that the tests are not affected by modified globals.
|
||||
resetTestGlobals()
|
||||
// initialize NSLock.
|
||||
initNSLock(false)
|
||||
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize server config. %s", err)
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://localhost"})
|
||||
@ -254,10 +298,6 @@ func TestListLocksHandler(t *testing.T) {
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := router.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
prefix string
|
||||
@ -308,7 +348,7 @@ func TestListLocksHandler(t *testing.T) {
|
||||
t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
adminRouter.ServeHTTP(rec, req)
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if test.expectedStatus != rec.Code {
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
|
||||
}
|
||||
@ -317,17 +357,11 @@ func TestListLocksHandler(t *testing.T) {
|
||||
|
||||
// Test for locks clear management REST API.
|
||||
func TestClearLocksHandler(t *testing.T) {
|
||||
// reset globals.
|
||||
// this is to make sure that the tests are not affected by modified globals.
|
||||
resetTestGlobals()
|
||||
// initialize NSLock.
|
||||
initNSLock(false)
|
||||
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize server config. %s", err)
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://localhost"})
|
||||
@ -336,10 +370,6 @@ func TestClearLocksHandler(t *testing.T) {
|
||||
}
|
||||
initGlobalAdminPeers(eps)
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := router.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
prefix string
|
||||
@ -390,7 +420,7 @@ func TestClearLocksHandler(t *testing.T) {
|
||||
t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
adminRouter.ServeHTTP(rec, req)
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if test.expectedStatus != rec.Code {
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
|
||||
}
|
||||
@ -556,36 +586,19 @@ func TestValidateHealQueryParams(t *testing.T) {
|
||||
|
||||
// TestListObjectsHeal - Test for ListObjectsHealHandler.
|
||||
func TestListObjectsHealHandler(t *testing.T) {
|
||||
rootPath, err := newTestConfig("us-east-1")
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize server config. %s", err)
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initializing objectLayer and corresponding []StorageAPI
|
||||
// since ListObjectsHeal() method requires it.
|
||||
objLayer, xlDirs, xlErr := prepareXL()
|
||||
if xlErr != nil {
|
||||
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
|
||||
}
|
||||
defer removeRoots(xlDirs)
|
||||
|
||||
err = objLayer.MakeBucket("mybucket")
|
||||
err = adminTestBed.objLayer.MakeBucket("mybucket")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to make bucket - %v", err)
|
||||
}
|
||||
|
||||
// Delete bucket after running all test cases.
|
||||
defer objLayer.DeleteBucket("mybucket")
|
||||
|
||||
// Make objLayer available to all internal services via globalObjectAPI.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := router.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
defer adminTestBed.objLayer.DeleteBucket("mybucket")
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
@ -695,7 +708,7 @@ func TestListObjectsHealHandler(t *testing.T) {
|
||||
t.Fatalf("Test %d - Failed to sign list objects needing heal request - %v", i+1, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
adminRouter.ServeHTTP(rec, req)
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if test.statusCode != rec.Code {
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
|
||||
}
|
||||
@ -704,36 +717,19 @@ func TestListObjectsHealHandler(t *testing.T) {
|
||||
|
||||
// TestHealBucketHandler - Test for HealBucketHandler.
|
||||
func TestHealBucketHandler(t *testing.T) {
|
||||
rootPath, err := newTestConfig("us-east-1")
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize server config. %s", err)
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initializing objectLayer and corresponding []StorageAPI
|
||||
// since MakeBucket() and DeleteBucket() methods requires it.
|
||||
objLayer, xlDirs, xlErr := prepareXL()
|
||||
if xlErr != nil {
|
||||
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
|
||||
}
|
||||
defer removeRoots(xlDirs)
|
||||
|
||||
err = objLayer.MakeBucket("mybucket")
|
||||
err = adminTestBed.objLayer.MakeBucket("mybucket")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to make bucket - %v", err)
|
||||
}
|
||||
|
||||
// Delete bucket after running all test cases.
|
||||
defer objLayer.DeleteBucket("mybucket")
|
||||
|
||||
// Make objLayer available to all internal services via globalObjectAPI.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := router.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
defer adminTestBed.objLayer.DeleteBucket("mybucket")
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
@ -771,7 +767,8 @@ func TestHealBucketHandler(t *testing.T) {
|
||||
|
||||
req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d - Failed to construct heal bucket request - %v", i+1, err)
|
||||
t.Fatalf("Test %d - Failed to construct heal bucket request - %v",
|
||||
i+1, err)
|
||||
}
|
||||
|
||||
req.Header.Set(minioAdminOpHeader, "bucket")
|
||||
@ -779,12 +776,14 @@ func TestHealBucketHandler(t *testing.T) {
|
||||
cred := serverConfig.GetCredential()
|
||||
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d - Failed to sign heal bucket request - %v", i+1, err)
|
||||
t.Fatalf("Test %d - Failed to sign heal bucket request - %v",
|
||||
i+1, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
adminRouter.ServeHTTP(rec, req)
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if test.statusCode != rec.Code {
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d",
|
||||
i+1, test.statusCode, rec.Code)
|
||||
}
|
||||
|
||||
}
|
||||
@ -792,29 +791,22 @@ func TestHealBucketHandler(t *testing.T) {
|
||||
|
||||
// TestHealObjectHandler - Test for HealObjectHandler.
|
||||
func TestHealObjectHandler(t *testing.T) {
|
||||
rootPath, err := newTestConfig("us-east-1")
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize server config. %s", err)
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
|
||||
// Initializing objectLayer and corresponding []StorageAPI
|
||||
// since MakeBucket(), PutObject() and DeleteBucket() method requires it.
|
||||
objLayer, xlDirs, xlErr := prepareXL()
|
||||
if xlErr != nil {
|
||||
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
|
||||
}
|
||||
defer removeRoots(xlDirs)
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Create an object myobject under bucket mybucket.
|
||||
bucketName := "mybucket"
|
||||
objName := "myobject"
|
||||
err = objLayer.MakeBucket(bucketName)
|
||||
err = adminTestBed.objLayer.MakeBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
|
||||
}
|
||||
|
||||
_, err = objLayer.PutObject(bucketName, objName, int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
|
||||
_, err = adminTestBed.objLayer.PutObject(bucketName, objName,
|
||||
int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create %s - %v", objName, err)
|
||||
}
|
||||
@ -823,16 +815,7 @@ func TestHealObjectHandler(t *testing.T) {
|
||||
defer func(objLayer ObjectLayer, bucketName, objName string) {
|
||||
objLayer.DeleteObject(bucketName, objName)
|
||||
objLayer.DeleteBucket(bucketName)
|
||||
}(objLayer, bucketName, objName)
|
||||
|
||||
// Make objLayer available to all internal services via globalObjectAPI.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := router.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
}(adminTestBed.objLayer, bucketName, objName)
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
@ -899,9 +882,42 @@ func TestHealObjectHandler(t *testing.T) {
|
||||
t.Fatalf("Test %d - Failed to sign heal object request - %v", i+1, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
adminRouter.ServeHTTP(rec, req)
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if test.statusCode != rec.Code {
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHealFormatHandler - test for HealFormatHandler.
|
||||
func TestHealFormatHandler(t *testing.T) {
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Prepare query params for heal-format mgmt REST API.
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("heal", "")
|
||||
req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct heal object request - %v", err)
|
||||
}
|
||||
|
||||
// Set x-minio-operation header to format.
|
||||
req.Header.Set(minioAdminOpHeader, "format")
|
||||
|
||||
// Sign the request using signature v4.
|
||||
cred := serverConfig.GetCredential()
|
||||
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to sign heal object request - %v", err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("Expected to succeed but failed with %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
@ -57,4 +57,6 @@ func registerAdminRouter(mux *router.Router) {
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "bucket").HandlerFunc(adminAPI.HealBucketHandler)
|
||||
// Heal Objects.
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
|
||||
// Heal Format.
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
|
||||
}
|
||||
|
@ -38,6 +38,7 @@ type remoteAdminClient struct {
|
||||
type adminCmdRunner interface {
|
||||
Restart() error
|
||||
ListLocks(bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error)
|
||||
ReInitDisks() error
|
||||
}
|
||||
|
||||
// Restart - Sends a message over channel to the go-routine
|
||||
@ -73,6 +74,20 @@ func (rc remoteAdminClient) ListLocks(bucket, prefix string, relTime time.Durati
|
||||
return reply.volLocks, nil
|
||||
}
|
||||
|
||||
// ReInitDisks - There is nothing to do here, heal format REST API
|
||||
// handler has already formatted and reinitialized the local disks.
|
||||
func (lc localAdminClient) ReInitDisks() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReInitDisks - Signals peers via RPC to reinitialize their disks and
|
||||
// object layer.
|
||||
func (rc remoteAdminClient) ReInitDisks() error {
|
||||
args := AuthRPCArgs{}
|
||||
reply := AuthRPCReply{}
|
||||
return rc.Call("Admin.ReInitDisks", &args, &reply)
|
||||
}
|
||||
|
||||
// adminPeer - represents an entity that implements Restart methods.
|
||||
type adminPeer struct {
|
||||
addr string
|
||||
@ -159,6 +174,8 @@ func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
|
||||
errs[0] = invokeServiceCmd(cps[0], cmd)
|
||||
}
|
||||
|
||||
// listPeerLocksInfo - fetch list of locks held on the given bucket,
|
||||
// matching prefix older than relTime from all peer servers.
|
||||
func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) {
|
||||
// Used to aggregate volume lock information from all nodes.
|
||||
allLocks := make([][]VolumeLockInfo, len(peers))
|
||||
@ -206,3 +223,21 @@ func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Dur
|
||||
}
|
||||
return groupedLockInfos, nil
|
||||
}
|
||||
|
||||
// reInitPeerDisks - reinitialize disks and object layer on peer servers to use the new format.
|
||||
func reInitPeerDisks(peers adminPeers) error {
|
||||
errs := make([]error, len(peers))
|
||||
|
||||
// Send ReInitDisks RPC call to all nodes.
|
||||
// for local adminPeer this is a no-op.
|
||||
wg := sync.WaitGroup{}
|
||||
for i, peer := range peers {
|
||||
wg.Add(1)
|
||||
go func(idx int, peer adminPeer) {
|
||||
defer wg.Done()
|
||||
errs[idx] = peer.cmdRunner.ReInitDisks()
|
||||
}(i, peer)
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/rpc"
|
||||
"time"
|
||||
|
||||
@ -25,6 +26,8 @@ import (
|
||||
|
||||
const adminPath = "/admin"
|
||||
|
||||
var errUnsupportedBackend = errors.New("not supported for non erasure-code backend")
|
||||
|
||||
// adminCmd - exports RPC methods for service status, stop and
|
||||
// restart commands.
|
||||
type adminCmd struct {
|
||||
@ -57,11 +60,51 @@ func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
|
||||
|
||||
// ListLocks - lists locks held by requests handled by this server instance.
|
||||
func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error {
|
||||
if err := query.IsAuthenticated(); err != nil {
|
||||
return err
|
||||
}
|
||||
volLocks := listLocksInfo(query.bucket, query.prefix, query.relTime)
|
||||
*reply = ListLocksReply{volLocks: volLocks}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReInitDisk - reinitialize storage disks and object layer to use the
|
||||
// new format.
|
||||
func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
|
||||
if err := args.IsAuthenticated(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !globalIsXL {
|
||||
return errUnsupportedBackend
|
||||
}
|
||||
|
||||
// Get the current object layer instance.
|
||||
objLayer := newObjectLayerFn()
|
||||
|
||||
// Initialize new disks to include the newly formatted disks.
|
||||
bootstrapDisks, err := initStorageDisks(globalEndpoints)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Initialize new object layer with newly formatted disks.
|
||||
newObjectAPI, err := newXLObjects(bootstrapDisks)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Replace object layer with newly formatted storage.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = newObjectAPI
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Shutdown storage belonging to old object layer instance.
|
||||
objLayer.Shutdown()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerAdminRPCRouter - registers RPC methods for service status,
|
||||
// stop and restart commands.
|
||||
func registerAdminRPCRouter(mux *router.Router) error {
|
||||
|
@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@ -61,6 +62,85 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdminRestart - test for Admin.Restart RPC service.
|
||||
func TestAdminRestart(t *testing.T) {
|
||||
testAdminCmd(restartCmd, t)
|
||||
}
|
||||
|
||||
// TestReInitDisks - test for Admin.ReInitDisks RPC service.
|
||||
func TestReInitDisks(t *testing.T) {
|
||||
// Reset global variables to start afresh.
|
||||
resetTestGlobals()
|
||||
|
||||
rootPath, err := newTestConfig("us-east-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize server config. %s", err)
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
|
||||
// Initializing objectLayer for HealFormatHandler.
|
||||
_, xlDirs, xlErr := initTestXLObjLayer()
|
||||
if xlErr != nil {
|
||||
t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
|
||||
}
|
||||
defer removeRoots(xlDirs)
|
||||
|
||||
// Set globalEndpoints for a single node XL setup.
|
||||
for _, xlDir := range xlDirs {
|
||||
globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
|
||||
}
|
||||
|
||||
// Setup admin rpc server for an XL backend.
|
||||
globalIsXL = true
|
||||
adminServer := adminCmd{}
|
||||
creds := serverConfig.GetCredential()
|
||||
args := LoginRPCArgs{
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
}
|
||||
reply := LoginRPCReply{}
|
||||
err = adminServer.Login(&args, &reply)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to login to admin server - %v", err)
|
||||
}
|
||||
|
||||
authArgs := AuthRPCArgs{
|
||||
AuthToken: reply.AuthToken,
|
||||
RequestTime: time.Now().UTC(),
|
||||
}
|
||||
authReply := AuthRPCReply{}
|
||||
|
||||
err = adminServer.ReInitDisks(&authArgs, &authReply)
|
||||
if err != nil {
|
||||
t.Errorf("Expected to pass, but failed with %v", err)
|
||||
}
|
||||
|
||||
// Negative test case with admin rpc server setup for FS.
|
||||
globalIsXL = false
|
||||
fsAdminServer := adminCmd{}
|
||||
fsArgs := LoginRPCArgs{
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
}
|
||||
fsReply := LoginRPCReply{}
|
||||
err = fsAdminServer.Login(&fsArgs, &fsReply)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to login to fs admin server - %v", err)
|
||||
}
|
||||
|
||||
authArgs = AuthRPCArgs{
|
||||
AuthToken: fsReply.AuthToken,
|
||||
RequestTime: time.Now().UTC(),
|
||||
}
|
||||
authReply = AuthRPCReply{}
|
||||
// Attempt ReInitDisks service on a FS backend.
|
||||
err = fsAdminServer.ReInitDisks(&authArgs, &authReply)
|
||||
if err != errUnsupportedBackend {
|
||||
t.Errorf("Expected to fail with %v, but received %v",
|
||||
errUnsupportedBackend, err)
|
||||
}
|
||||
}
|
||||
|
@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"net/url"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
@ -70,6 +71,9 @@ var (
|
||||
// Indicates if the running minio server is distributed setup.
|
||||
globalIsDistXL = false
|
||||
|
||||
// Indicates if the running minio server is an erasure-code backend.
|
||||
globalIsXL = false
|
||||
|
||||
// This flag is set to 'true' by default, it is set to `false`
|
||||
// when MINIO_BROWSER env is set to 'off'.
|
||||
globalIsBrowserEnabled = !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off")
|
||||
@ -112,6 +116,9 @@ var (
|
||||
// Secret key passed from the environment
|
||||
globalEnvSecretKey = os.Getenv("MINIO_SECRET_KEY")
|
||||
|
||||
// url.URL endpoints of disks that belong to the object storage.
|
||||
globalEndpoints = []*url.URL{}
|
||||
|
||||
// Add new variable global values here.
|
||||
)
|
||||
|
||||
|
@ -418,6 +418,12 @@ func serverMain(c *cli.Context) {
|
||||
fatalIf(initDsyncNodes(endpoints), "Unable to initialize distributed locking clients")
|
||||
}
|
||||
|
||||
// Set globalIsXL if erasure code backend is about to be
|
||||
// initialized for the given endpoints.
|
||||
if len(endpoints) > 1 {
|
||||
globalIsXL = true
|
||||
}
|
||||
|
||||
// Initialize name space lock.
|
||||
initNSLock(globalIsDistXL)
|
||||
|
||||
@ -453,6 +459,9 @@ func serverMain(c *cli.Context) {
|
||||
fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server.")
|
||||
}()
|
||||
|
||||
// Set endpoints of []*url.URL type to globalEndpoints.
|
||||
globalEndpoints = endpoints
|
||||
|
||||
newObject, err := newObjectLayer(srvConfig)
|
||||
fatalIf(err, "Initializing object layer failed")
|
||||
|
||||
|
@ -488,6 +488,14 @@ func resetGlobalEventnotify() {
|
||||
globalEventNotifier = nil
|
||||
}
|
||||
|
||||
func resetGlobalEndpoints() {
|
||||
globalEndpoints = []*url.URL{}
|
||||
}
|
||||
|
||||
func resetGlobalIsXL() {
|
||||
globalIsXL = false
|
||||
}
|
||||
|
||||
// Resets all the globals used modified in tests.
|
||||
// Resetting ensures that the changes made to globals by one test doesn't affect others.
|
||||
func resetTestGlobals() {
|
||||
@ -501,6 +509,10 @@ func resetTestGlobals() {
|
||||
resetGlobalNSLock()
|
||||
// Reset global event notifier.
|
||||
resetGlobalEventnotify()
|
||||
// Reset global endpoints.
|
||||
resetGlobalEndpoints()
|
||||
// Reset global isXL flag.
|
||||
resetGlobalIsXL()
|
||||
}
|
||||
|
||||
// Configure the server for the test run.
|
||||
|
@ -163,6 +163,14 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
|
||||
// Shutdown function for object storage interface.
|
||||
func (xl xlObjects) Shutdown() error {
|
||||
// Add any object layer shutdown activities here.
|
||||
for _, disk := range xl.storageDisks {
|
||||
// This closes storage rpc client connections if any.
|
||||
// Otherwise this is a no-op.
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
disk.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -38,8 +38,11 @@ func main() {
|
||||
|
||||
| Service operations|LockInfo operations|Healing operations|
|
||||
|:---|:---|:---|
|
||||
|[`ServiceStatus`](#ServiceStatus)| | |
|
||||
|[`ServiceRestart`](#ServiceRestart)| | |
|
||||
|[`ServiceStatus`](#ServiceStatus)| [`ListLocks`](#ListLocks)| [`ListObjectsHeal`](#ListObjectsHeal)|
|
||||
|[`ServiceRestart`](#ServiceRestart)| [`ClearLocks`](#ClearLocks)| [`ListBucketsHeal`](#ListBucketsHeal)|
|
||||
| | |[`HealBucket`](#HealBucket) |
|
||||
| | |[`HealObject`](#HealObject)|
|
||||
| | |[`HealFormat`](#HealFormat)|
|
||||
|
||||
## 1. Constructor
|
||||
<a name="Minio"></a>
|
||||
@ -185,14 +188,14 @@ __Example__
|
||||
}
|
||||
```
|
||||
|
||||
<a name="ListBucketsList"></a>
|
||||
### ListBucketsList() error
|
||||
<a name="ListBucketsHeal"></a>
|
||||
### ListBucketsHeal() error
|
||||
If successful returns information on the list of buckets that need healing.
|
||||
|
||||
__Example__
|
||||
|
||||
``` go
|
||||
// List buckets that need healing
|
||||
// List buckets that need healing
|
||||
healBucketsList, err := madmClnt.ListBucketsHeal()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
@ -244,3 +247,19 @@ __Example__
|
||||
log.Println("successfully healed mybucket/myobject")
|
||||
|
||||
```
|
||||
|
||||
<a name="HealFormat"></a>
|
||||
### HealFormat() error
|
||||
Heal storage format on available disks. This is used when disks were replaced or were found with missing format. This is supported only for erasure-coded backend.
|
||||
|
||||
__Example__
|
||||
|
||||
``` go
|
||||
err := madmClnt.HealFormat()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
log.Println("successfully healed storage format on available disks.")
|
||||
|
||||
```
|
||||
|
49
pkg/madmin/examples/heal-format.go
Normal file
49
pkg/madmin/examples/heal-format.go
Normal file
@ -0,0 +1,49 @@
|
||||
// +build ignore
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
|
||||
// dummy values, please replace them with original values.
|
||||
|
||||
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
|
||||
// dummy values, please replace them with original values.
|
||||
|
||||
// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
|
||||
// New returns an Minio Admin client object.
|
||||
madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// Heal storage format on available disks.
|
||||
err = madmClnt.HealFormat()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
log.Println("successfully healed storage format on available disks.")
|
||||
}
|
@ -403,3 +403,32 @@ func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HealFormat - heal storage format on available disks.
|
||||
func (adm *AdminClient) HealFormat() error {
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("heal", "")
|
||||
|
||||
// Set x-minio-operation to format.
|
||||
hdrs := make(http.Header)
|
||||
hdrs.Set(minioAdminOpHeader, "format")
|
||||
|
||||
reqData := requestData{
|
||||
queryValues: queryVal,
|
||||
customHeaders: hdrs,
|
||||
}
|
||||
|
||||
// Execute POST on /?heal to heal storage format.
|
||||
resp, err := adm.executeMethod("POST", reqData)
|
||||
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return errors.New("Got HTTP Status: " + resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user