Bring in safe mode support (#8478)

This PR refactors object layer handling such
that upon failure in sub-system initialization
server reaches a stage of safe-mode operation
wherein only certain API operations are enabled
and available.

This allows for fixing many scenarios such as

 - incorrect configuration in vault, etcd,
   notification targets
 - missing files, incomplete config migrations
   unable to read encrypted content etc
 - any other issues related to notification,
   policies, lifecycle etc
This commit is contained in:
Harshavardhana
2019-11-09 09:27:23 -08:00
committed by kannappanr
parent 1c90a6bd49
commit 822eb5ddc7
41 changed files with 1129 additions and 830 deletions

View File

@@ -222,9 +222,22 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
logger.FatalIf(err, "Unable to initialize gateway backend")
}
// Re-enable logging
logger.Disable = false
// Once endpoints are finalized, initialize the new object api in safe mode.
globalObjLayerMutex.Lock()
globalSafeMode = true
globalObjectAPI = newObject
globalObjLayerMutex.Unlock()
// Populate existing buckets to the etcd backend
if globalDNSConfig != nil {
initFederatorBackend(newObject)
buckets, err := newObject.ListBuckets(context.Background())
if err != nil {
logger.Fatal(err, "Unable to list buckets")
}
initFederatorBackend(buckets, newObject)
}
// Migrate all backend configs to encrypted backend, also handles rotation as well.
@@ -233,16 +246,15 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
logger.FatalIf(handleEncryptedConfigBackend(newObject, enableConfigOps),
"Unable to handle encrypted backend for config, iam and policies")
// Calls all New() for all sub-systems.
newAllSubsystems()
// **** WARNING ****
// Migrating to encrypted backend should happen before initialization of any
// sub-systems, make sure that we do not move the above codeblock elsewhere.
if enableConfigOps {
// Create a new config system.
globalConfigSys = NewConfigSys()
// Load globalServerConfig from disk
logger.LogIf(context.Background(), globalConfigSys.Init(newObject))
logger.FatalIf(globalConfigSys.Init(newObject), "Unable to initialize config system")
// Start watching disk for reloading config, this
// is only enabled for "NAS" gateway.
@@ -261,32 +273,20 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
"Unable to handle encrypted backend for iam and policies")
}
if globalCacheConfig.Enabled {
// initialize the new disk cache objects.
globalCacheObjectAPI, err = newServerCacheObjects(context.Background(), globalCacheConfig)
logger.FatalIf(err, "Unable to initialize disk caching")
}
// Re-enable logging
logger.Disable = false
// Create new IAM system.
globalIAMSys = NewIAMSys()
if enableIAMOps {
// Initialize IAM sys.
logger.LogIf(context.Background(), globalIAMSys.Init(newObject))
logger.FatalIf(globalIAMSys.Init(newObject), "Unable to initialize IAM system")
}
// Create new policy system.
globalPolicySys = NewPolicySys()
if globalCacheConfig.Enabled {
// initialize the new disk cache objects.
var cacheAPI CacheObjectLayer
cacheAPI, err = newServerCacheObjects(context.Background(), globalCacheConfig)
logger.FatalIf(err, "Unable to initialize disk caching")
// Create new lifecycle system.
globalLifecycleSys = NewLifecycleSys()
// Create new notification system.
globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
if err != nil {
logger.FatalIf(err, "Unable to initialize notification system")
globalObjLayerMutex.Lock()
globalCacheObjectAPI = cacheAPI
globalObjLayerMutex.Unlock()
}
// Verify if object layer supports
@@ -294,11 +294,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
// - compression
verifyObjectLayerFeatures("gateway "+gatewayName, newObject)
// Once endpoints are finalized, initialize the new object api.
globalObjLayerMutex.Lock()
globalObjectAPI = newObject
globalObjLayerMutex.Unlock()
// Prints the formatted startup message once object layer is initialized.
if !globalCLIContext.Quiet {
mode := globalMinioModeGatewayPrefix + gatewayName
@@ -314,6 +309,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
}
// Disable safe mode operation, after all initialization is over.
globalObjLayerMutex.Lock()
globalSafeMode = false
globalObjLayerMutex.Unlock()
// Set uptime time after object layer has initialized.
globalBootTime = UTCNow()