mirror of
https://github.com/minio/minio.git
synced 2025-11-20 09:56:07 -05:00
Bring in safe mode support (#8478)
This PR refactors object layer handling such that upon failure in sub-system initialization server reaches a stage of safe-mode operation wherein only certain API operations are enabled and available. This allows for fixing many scenarios such as - incorrect configuration in vault, etcd, notification targets - missing files, incomplete config migrations unable to read encrypted content etc - any other issues related to notification, policies, lifecycle etc
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
1c90a6bd49
commit
822eb5ddc7
@@ -180,53 +180,99 @@ func serverHandleEnvVars() {
|
||||
handleCommonEnvVars()
|
||||
}
|
||||
|
||||
func initAllSubsystems(newObject ObjectLayer) {
|
||||
func newAllSubsystems() {
|
||||
// Create new notification system and initialize notification targets
|
||||
globalNotificationSys = NewNotificationSys(globalEndpoints)
|
||||
|
||||
// Create a new config system.
|
||||
globalConfigSys = NewConfigSys()
|
||||
|
||||
// Initialize config system.
|
||||
if err := globalConfigSys.Init(newObject); err != nil {
|
||||
logger.Fatal(err, "Unable to initialize config system")
|
||||
}
|
||||
|
||||
// Create new IAM system.
|
||||
globalIAMSys = NewIAMSys()
|
||||
|
||||
if err := globalIAMSys.Init(newObject); err != nil {
|
||||
logger.Fatal(err, "Unable to initialize IAM system")
|
||||
// Create new policy system.
|
||||
globalPolicySys = NewPolicySys()
|
||||
|
||||
// Create new lifecycle system.
|
||||
globalLifecycleSys = NewLifecycleSys()
|
||||
}
|
||||
|
||||
func initSafeModeInit(buckets []BucketInfo) (err error) {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
// Enable logger
|
||||
logger.Disable = false
|
||||
|
||||
// Prints the formatted startup message in safe mode operation.
|
||||
printStartupSafeModeMessage(getAPIEndpoints(), err)
|
||||
|
||||
// Initialization returned error reaching safe mode and
|
||||
// not proceeding waiting for admin action.
|
||||
handleSignals()
|
||||
}
|
||||
}()
|
||||
|
||||
newObject := newObjectLayerWithoutSafeModeFn()
|
||||
|
||||
// Calls New() for all sub-systems.
|
||||
newAllSubsystems()
|
||||
|
||||
// Migrate all backend configs to encrypted backend, also handles rotation as well.
|
||||
if err = handleEncryptedConfigBackend(newObject, true); err != nil {
|
||||
return fmt.Errorf("Unable to handle encrypted backend for config, iam and policies: %v", err)
|
||||
}
|
||||
|
||||
buckets, err := newObject.ListBuckets(context.Background())
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to list buckets")
|
||||
// **** WARNING ****
|
||||
// Migrating to encrypted backend should happen before initialization of any
|
||||
// sub-systems, make sure that we do not move the above codeblock elsewhere.
|
||||
|
||||
// Validate and initialize all subsystems.
|
||||
if err = initAllSubsystems(buckets, newObject); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create new notification system and initialize notification targets
|
||||
globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize notification targets")
|
||||
if globalEtcdClient != nil {
|
||||
// **** WARNING ****
|
||||
// Migrating to encrypted backend on etcd should happen before initialization of
|
||||
// IAM sub-systems, make sure that we do not move the above codeblock elsewhere.
|
||||
if err = migrateIAMConfigsEtcdToEncrypted(globalEtcdClient); err != nil {
|
||||
return fmt.Errorf("Unable to handle encrypted backend for iam and policies: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func initAllSubsystems(buckets []BucketInfo, newObject ObjectLayer) (err error) {
|
||||
// Initialize config system.
|
||||
if err = globalConfigSys.Init(newObject); err != nil {
|
||||
return fmt.Errorf("Unable to initialize config system: %v", err)
|
||||
}
|
||||
|
||||
if err = globalNotificationSys.AddNotificationTargetsFromConfig(globalServerConfig); err != nil {
|
||||
return fmt.Errorf("Unable to initialize notification target(s) from config: %v", err)
|
||||
}
|
||||
|
||||
if err = globalIAMSys.Init(newObject); err != nil {
|
||||
return fmt.Errorf("Unable to initialize IAM system: %v", err)
|
||||
}
|
||||
|
||||
// Initialize notification system.
|
||||
if err = globalNotificationSys.Init(buckets, newObject); err != nil {
|
||||
logger.Fatal(err, "Unable to initialize notification system")
|
||||
return fmt.Errorf("Unable to initialize notification system: %v", err)
|
||||
}
|
||||
|
||||
// Create new policy system.
|
||||
globalPolicySys = NewPolicySys()
|
||||
|
||||
// Initialize policy system.
|
||||
if err = globalPolicySys.Init(buckets, newObject); err != nil {
|
||||
logger.Fatal(err, "Unable to initialize policy system")
|
||||
return fmt.Errorf("Unable to initialize policy system; %v", err)
|
||||
}
|
||||
|
||||
// Create new lifecycle system.
|
||||
globalLifecycleSys = NewLifecycleSys()
|
||||
|
||||
// Initialize lifecycle system.
|
||||
if err = globalLifecycleSys.Init(buckets, newObject); err != nil {
|
||||
logger.Fatal(err, "Unable to initialize lifecycle system")
|
||||
return fmt.Errorf("Unable to initialize lifecycle system: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// serverMain handler called for 'minio server' command.
|
||||
@@ -327,42 +373,33 @@ func serverMain(ctx *cli.Context) {
|
||||
globalTLSCerts.Stop()
|
||||
|
||||
globalHTTPServer.Shutdown()
|
||||
logger.FatalIf(err, "Unable to initialize backend")
|
||||
}
|
||||
|
||||
// Populate existing buckets to the etcd backend
|
||||
if globalDNSConfig != nil {
|
||||
initFederatorBackend(newObject)
|
||||
logger.Fatal(err, "Unable to initialize backend")
|
||||
}
|
||||
|
||||
// Re-enable logging
|
||||
logger.Disable = false
|
||||
|
||||
// Migrate all backend configs to encrypted backend, also handles rotation as well.
|
||||
logger.FatalIf(handleEncryptedConfigBackend(newObject, true),
|
||||
"Unable to handle encrypted backend for config, iam and policies")
|
||||
// Once endpoints are finalized, initialize the new object api in safe mode.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalSafeMode = true
|
||||
globalObjectAPI = newObject
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// **** WARNING ****
|
||||
// Migrating to encrypted backend should happen before initialization of any
|
||||
// sub-systems, make sure that we do not move the above codeblock elsewhere.
|
||||
|
||||
// Validate and initialize all subsystems.
|
||||
initAllSubsystems(newObject)
|
||||
|
||||
if globalEtcdClient != nil {
|
||||
// **** WARNING ****
|
||||
// Migrating to encrypted backend on etcd should happen before initialization of
|
||||
// IAM sub-systems, make sure that we do not move the above codeblock elsewhere.
|
||||
logger.FatalIf(migrateIAMConfigsEtcdToEncrypted(globalEtcdClient),
|
||||
"Unable to handle encrypted backend for iam and policies")
|
||||
buckets, err := newObject.ListBuckets(context.Background())
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to list buckets")
|
||||
}
|
||||
|
||||
if globalCacheConfig.Enabled {
|
||||
logger.StartupMessage(color.Red(color.Bold("Disk caching is recommended only for gateway deployments")))
|
||||
// Populate existing buckets to the etcd backend
|
||||
if globalDNSConfig != nil {
|
||||
initFederatorBackend(buckets, newObject)
|
||||
}
|
||||
|
||||
// initialize the new disk cache objects.
|
||||
globalCacheObjectAPI, err = newServerCacheObjects(context.Background(), globalCacheConfig)
|
||||
logger.FatalIf(err, "Unable to initialize disk caching")
|
||||
initSafeModeInit(buckets)
|
||||
|
||||
if globalCacheConfig.Enabled {
|
||||
msg := color.RedBold("Disk caching is disabled in 'server' mode, 'caching' is only supported in gateway deployments")
|
||||
logger.StartupMessage(msg)
|
||||
}
|
||||
|
||||
initDailyLifecycle()
|
||||
@@ -373,14 +410,17 @@ func serverMain(ctx *cli.Context) {
|
||||
initGlobalHeal()
|
||||
}
|
||||
|
||||
globalObjectAPI = newObject
|
||||
// Disable safe mode operation, after all initialization is over.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalSafeMode = false
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
printStartupMessage(getAPIEndpoints())
|
||||
|
||||
if globalActiveCred.Equal(auth.DefaultCredentials) {
|
||||
msg := fmt.Sprintf("Detected default credentials '%s', please change the credentials immediately using 'MINIO_ACCESS_KEY' and 'MINIO_SECRET_KEY'", globalActiveCred)
|
||||
logger.StartupMessage(color.Red(color.Bold(msg)))
|
||||
logger.StartupMessage(color.RedBold(msg))
|
||||
}
|
||||
|
||||
// Set uptime time after object layer has initialized.
|
||||
|
||||
Reference in New Issue
Block a user