Bring in safe mode support (#8478)

This PR refactors object layer handling such
that upon failure in sub-system initialization
server reaches a stage of safe-mode operation
wherein only certain API operations are enabled
and available.

This allows for fixing many scenarios such as

 - incorrect configuration in vault, etcd,
   notification targets
 - missing files, incomplete config migrations
   unable to read encrypted content etc
 - any other issues related to notification,
   policies, lifecycle etc
This commit is contained in:
Harshavardhana
2019-11-09 09:27:23 -08:00
committed by kannappanr
parent 1c90a6bd49
commit 822eb5ddc7
41 changed files with 1129 additions and 830 deletions

View File

@@ -31,12 +31,13 @@ import (
// Documentation links, these are part of message printing code.
const (
mcQuickStartGuide = "https://docs.min.io/docs/minio-client-quickstart-guide"
goQuickStartGuide = "https://docs.min.io/docs/golang-client-quickstart-guide"
jsQuickStartGuide = "https://docs.min.io/docs/javascript-client-quickstart-guide"
javaQuickStartGuide = "https://docs.min.io/docs/java-client-quickstart-guide"
pyQuickStartGuide = "https://docs.min.io/docs/python-client-quickstart-guide"
dotnetQuickStartGuide = "https://docs.min.io/docs/dotnet-client-quickstart-guide"
mcQuickStartGuide = "https://docs.min.io/docs/minio-client-quickstart-guide"
mcAdminQuickStartGuide = "https://docs.min.io/docs/minio-admin-complete-guide.html"
goQuickStartGuide = "https://docs.min.io/docs/golang-client-quickstart-guide"
jsQuickStartGuide = "https://docs.min.io/docs/javascript-client-quickstart-guide"
javaQuickStartGuide = "https://docs.min.io/docs/java-client-quickstart-guide"
pyQuickStartGuide = "https://docs.min.io/docs/python-client-quickstart-guide"
dotnetQuickStartGuide = "https://docs.min.io/docs/dotnet-client-quickstart-guide"
)
// generates format string depending on the string length and padding.
@@ -45,16 +46,75 @@ func getFormatStr(strLen int, padding int) string {
return "%" + formatStr
}
// Prints the formatted startup message.
func printStartupMessage(apiEndPoints []string) {
func printStartupSafeModeMessage(apiEndpoints []string, err error) {
logStartupMessage(color.RedBold("Server startup failed with '%v'", err))
logStartupMessage(color.RedBold("Server switching to safe mode"))
logStartupMessage(color.RedBold("Please use 'mc admin' commands to fix this issue"))
strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
// If cache layer is enabled, print cache capacity.
if globalCacheObjectAPI != nil {
printCacheStorageInfo(globalCacheObjectAPI.StorageInfo(context.Background()))
// Object layer is initialized then print StorageInfo in safe mode.
objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI != nil {
if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background())); msg != "" {
logStartupMessage(msg)
}
}
// Get saved credentials.
cred := globalActiveCred
// Get saved region.
region := globalServerRegion
strippedAPIEndpoints := stripStandardPorts(apiEndpoints)
apiEndpointStr := strings.Join(strippedAPIEndpoints, " ")
// Colorize the message and print.
logStartupMessage(color.Red("Endpoint: ") + color.Bold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
if color.IsTerminal() && !globalCLIContext.Anonymous {
logStartupMessage(color.Red("AccessKey: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
logStartupMessage(color.Red("SecretKey: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
if region != "" {
logStartupMessage(color.Red("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
}
}
// Prints `mc` cli configuration message chooses
// first endpoint as default.
alias := "myminio"
endPoint := strippedAPIEndpoints[0]
// Configure 'mc', following block prints platform specific information for minio client admin commands.
if color.IsTerminal() {
logStartupMessage(color.RedBold("\nCommand-line Access: ") + mcAdminQuickStartGuide)
if runtime.GOOS == globalWindowsOSName {
mcMessage := fmt.Sprintf("> mc.exe config host add %s %s %s %s --api s3v4", alias,
endPoint, cred.AccessKey, cred.SecretKey)
logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
mcMessage = fmt.Sprintf("> mc.exe admin --help")
logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
} else {
mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s --api s3v4", alias,
endPoint, cred.AccessKey, cred.SecretKey)
logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
mcMessage = fmt.Sprintf("$ mc admin --help")
logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
}
}
}
// Prints the formatted startup message.
func printStartupMessage(apiEndpoints []string) {
strippedAPIEndpoints := stripStandardPorts(apiEndpoints)
// If cache layer is enabled, print cache capacity.
cachedObjAPI := newCachedObjectLayerFn()
if cachedObjAPI != nil {
printCacheStorageInfo(cachedObjAPI.StorageInfo(context.Background()))
}
// Object layer is initialized then print StorageInfo.
objAPI := globalObjectAPI
objAPI := newObjectLayerFn()
if objAPI != nil {
printStorageInfo(objAPI.StorageInfo(context.Background()))
}
@@ -183,6 +243,16 @@ func printObjectAPIMsg() {
logStartupMessage(color.Blue(" .NET: ") + fmt.Sprintf(getFormatStr(len(dotnetQuickStartGuide), 6), dotnetQuickStartGuide))
}
// Get formatted disk/storage info message.
func getStorageInfoMsgSafeMode(storageInfo StorageInfo) string {
var msg string
if storageInfo.Backend.Type == BackendErasure {
diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks.Sum(), storageInfo.Backend.OfflineDisks.Sum())
msg += color.Red("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo)
}
return msg
}
// Get formatted disk/storage info message.
func getStorageInfoMsg(storageInfo StorageInfo) string {
var msg string