Bring in safe mode support (#8478)

This PR refactors object layer handling such
that upon failure in sub-system initialization
server reaches a stage of safe-mode operation
wherein only certain API operations are enabled
and available.

This allows for fixing many scenarios such as

 - incorrect configuration in vault, etcd,
   notification targets
 - missing files, incomplete config migrations
   unable to read encrypted content etc
 - any other issues related to notification,
   policies, lifecycle etc

cmd/crypto/config.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"errors"
 	"fmt"
+	"reflect"
 	"strconv"
 
 	"github.com/minio/minio/cmd/config"
@@ -112,6 +113,12 @@ const (
 	EnvKMSVaultNamespace = "MINIO_KMS_VAULT_NAMESPACE"
 )
 
+var defaultCfg = VaultConfig{
+	Auth: VaultAuth{
+		Type: "approle",
+	},
+}
+
 // LookupConfig extracts the KMS configuration provided by environment
 // variables and merge them with the provided KMS configuration. The
 // merging follows the following rules:
@@ -139,7 +146,7 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) {
 			return kmsCfg, err
 		}
 	}
-	if !kmsCfg.Vault.IsEmpty() {
+	if kmsCfg.Vault.Enabled {
 		return kmsCfg, nil
 	}
 	stateBool, err := config.ParseBool(env.Get(EnvKMSVaultState, kvs.Get(config.State)))
@@ -166,23 +173,30 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) {
 	vcfg.Endpoint = endpointStr
 	vcfg.CAPath = env.Get(EnvKMSVaultCAPath, kvs.Get(KMSVaultCAPath))
 	vcfg.Auth.Type = env.Get(EnvKMSVaultAuthType, kvs.Get(KMSVaultAuthType))
+	if vcfg.Auth.Type == "" {
+		vcfg.Auth.Type = "approle"
+	}
 	vcfg.Auth.AppRole.ID = env.Get(EnvKMSVaultAppRoleID, kvs.Get(KMSVaultAppRoleID))
 	vcfg.Auth.AppRole.Secret = env.Get(EnvKMSVaultAppSecretID, kvs.Get(KMSVaultAppRoleSecret))
 	vcfg.Key.Name = env.Get(EnvKMSVaultKeyName, kvs.Get(KMSVaultKeyName))
 	vcfg.Namespace = env.Get(EnvKMSVaultNamespace, kvs.Get(KMSVaultNamespace))
-	keyVersion := env.Get(EnvKMSVaultKeyVersion, kvs.Get(KMSVaultKeyVersion))
-
-	if keyVersion != "" {
+	if keyVersion := env.Get(EnvKMSVaultKeyVersion, kvs.Get(KMSVaultKeyVersion)); keyVersion != "" {
 		vcfg.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
 			return kmsCfg, fmt.Errorf("Unable to parse VaultKeyVersion value (`%s`)", keyVersion)
 		}
 	}
 
+	if reflect.DeepEqual(vcfg, defaultCfg) {
+		return kmsCfg, nil
+	}
+
+	// Verify all the proper settings.
 	if err = vcfg.Verify(); err != nil {
 		return kmsCfg, err
 	}
 
+	vcfg.Enabled = true
 	kmsCfg.Vault = vcfg
 	return kmsCfg, nil
 }
@@ -191,7 +205,7 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) {
 func NewKMS(cfg KMSConfig) (kms KMS, err error) {
 	// Lookup KMS master keys - only available through ENV.
 	if masterKeyLegacy := env.Get(EnvKMSMasterKeyLegacy, ""); len(masterKeyLegacy) != 0 {
-		if !cfg.Vault.IsEmpty() { // Vault and KMS master key provided
+		if cfg.Vault.Enabled { // Vault and KMS master key provided
 			return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
 		}
 		kms, err = ParseMasterKey(masterKeyLegacy)
@@ -199,15 +213,14 @@ func NewKMS(cfg KMSConfig) (kms KMS, err error) {
 			return kms, err
 		}
 	} else if masterKey := env.Get(EnvKMSMasterKey, ""); len(masterKey) != 0 {
-		if !cfg.Vault.IsEmpty() { // Vault and KMS master key provided
+		if cfg.Vault.Enabled { // Vault and KMS master key provided
 			return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
 		}
 		kms, err = ParseMasterKey(masterKey)
 		if err != nil {
 			return kms, err
 		}
-	}
-	if !cfg.Vault.IsEmpty() {
+	} else if cfg.Vault.Enabled {
 		kms, err = NewVault(cfg.Vault)
 		if err != nil {
 			return kms, err

cmd/crypto/legacy.go

@@ -18,6 +18,7 @@ package crypto
 
 import (
 	"fmt"
+	"reflect"
 	"strconv"
 
 	"github.com/minio/minio/cmd/config"
@@ -40,40 +41,40 @@ const (
 )
 
 const (
-	// EnvVaultEndpoint is the environment variable used to specify
+	// EnvLegacyVaultEndpoint is the environment variable used to specify
 	// the vault HTTPS endpoint.
-	EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
+	EnvLegacyVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
 
-	// EnvVaultAuthType is the environment variable used to specify
+	// EnvLegacyVaultAuthType is the environment variable used to specify
 	// the authentication type for vault.
-	EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
+	EnvLegacyVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
 
-	// EnvVaultAppRoleID is the environment variable used to specify
+	// EnvLegacyVaultAppRoleID is the environment variable used to specify
 	// the vault AppRole ID.
-	EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
+	EnvLegacyVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
 
-	// EnvVaultAppSecretID is the environment variable used to specify
+	// EnvLegacyVaultAppSecretID is the environment variable used to specify
 	// the vault AppRole secret corresponding to the AppRole ID.
-	EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
+	EnvLegacyVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
 
-	// EnvVaultKeyVersion is the environment variable used to specify
+	// EnvLegacyVaultKeyVersion is the environment variable used to specify
 	// the vault key version.
-	EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
+	EnvLegacyVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
 
-	// EnvVaultKeyName is the environment variable used to specify
+	// EnvLegacyVaultKeyName is the environment variable used to specify
 	// the vault named key-ring. In the S3 context it's referred as
 	// customer master key ID (CMK-ID).
-	EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
+	EnvLegacyVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
 
-	// EnvVaultCAPath is the environment variable used to specify the
+	// EnvLegacyVaultCAPath is the environment variable used to specify the
 	// path to a directory of PEM-encoded CA cert files. These CA cert
 	// files are used to authenticate MinIO to Vault over mTLS.
-	EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
+	EnvLegacyVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
 
-	// EnvVaultNamespace is the environment variable used to specify
+	// EnvLegacyVaultNamespace is the environment variable used to specify
 	// vault namespace. The vault namespace is used if the enterprise
 	// version of Hashicorp Vault is used.
-	EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
+	EnvLegacyVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
 )
 
 // SetKMSConfig helper to migrate from older KMSConfig to new KV.
@@ -93,7 +94,7 @@ func SetKMSConfig(s config.Config, cfg KMSConfig) {
 		KMSVaultKeyVersion:    strconv.Itoa(cfg.Vault.Key.Version),
 		KMSVaultNamespace:     cfg.Vault.Namespace,
 		config.State: func() string {
-			if !cfg.Vault.IsEmpty() {
+			if cfg.Vault.Endpoint != "" {
 				return config.StateOn
 			}
 			return config.StateOff
@@ -141,7 +142,7 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
 		return cfg, nil
 	}
 
-	endpointStr := env.Get(EnvKMSVaultEndpoint, kvs.Get(KMSVaultEndpoint))
+	endpointStr := env.Get(EnvLegacyVaultEndpoint, "")
 	if endpointStr != "" {
 		// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
 		endpoint, err := xnet.ParseHTTPURL(endpointStr)
@@ -152,25 +153,31 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
 	}
 
 	cfg.Vault.Endpoint = endpointStr
-	cfg.Vault.CAPath = env.Get(EnvVaultCAPath, kvs.Get(KMSVaultCAPath))
-	cfg.Vault.Auth.Type = env.Get(EnvVaultAuthType, kvs.Get(KMSVaultAuthType))
-	cfg.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, kvs.Get(KMSVaultAppRoleID))
-	cfg.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, kvs.Get(KMSVaultAppRoleSecret))
-	cfg.Vault.Key.Name = env.Get(EnvVaultKeyName, kvs.Get(KMSVaultKeyName))
-	cfg.Vault.Namespace = env.Get(EnvVaultNamespace, kvs.Get(KMSVaultNamespace))
-	keyVersion := env.Get(EnvVaultKeyVersion, kvs.Get(KMSVaultKeyVersion))
-
-	if keyVersion != "" {
+	cfg.Vault.CAPath = env.Get(EnvLegacyVaultCAPath, "")
+	cfg.Vault.Auth.Type = env.Get(EnvLegacyVaultAuthType, "")
+	if cfg.Vault.Auth.Type == "" {
+		cfg.Vault.Auth.Type = "approle"
+	}
+	cfg.Vault.Auth.AppRole.ID = env.Get(EnvLegacyVaultAppRoleID, "")
+	cfg.Vault.Auth.AppRole.Secret = env.Get(EnvLegacyVaultAppSecretID, "")
+	cfg.Vault.Key.Name = env.Get(EnvLegacyVaultKeyName, "")
+	cfg.Vault.Namespace = env.Get(EnvLegacyVaultNamespace, "")
+	if keyVersion := env.Get(EnvLegacyVaultKeyVersion, ""); keyVersion != "" {
 		cfg.Vault.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
 			return cfg, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
-				EnvVaultKeyVersion, keyVersion)
+				EnvLegacyVaultKeyVersion, keyVersion)
 		}
 	}
 
+	if reflect.DeepEqual(cfg.Vault, defaultCfg) {
+		return cfg, nil
+	}
+
 	if err = cfg.Vault.Verify(); err != nil {
 		return cfg, err
 	}
 
+	cfg.Vault.Enabled = true
 	return cfg, nil
 }

cmd/crypto/vault.go

@@ -51,6 +51,7 @@ type VaultAppRole struct {
 
 // VaultConfig represents vault configuration.
 type VaultConfig struct {
+	Enabled   bool      `json:"-"`
 	Endpoint  string    `json:"endpoint"` // The vault API endpoint as URL
 	CAPath    string    `json:"-"`        // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
 	Auth      VaultAuth `json:"auth"`     // The vault authentication configuration
@@ -68,24 +69,10 @@ type vaultService struct {
 
 var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
 
-// empty/default vault configuration used to check whether a particular is empty.
-var emptyVaultConfig = VaultConfig{
-	Auth: VaultAuth{
-		Type: "approle",
-	},
-}
-
-// IsEmpty returns true if the vault config struct is an
-// empty configuration.
-func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }
-
 // Verify returns a nil error if the vault configuration
 // is valid. A valid configuration is either empty or
 // contains valid non-default values.
 func (v *VaultConfig) Verify() (err error) {
-	if v.IsEmpty() {
-		return // an empty configuration is valid
-	}
 	switch {
 	case v.Endpoint == "":
 		err = errors.New("crypto: missing hashicorp vault endpoint")
@@ -107,8 +94,8 @@ func (v *VaultConfig) Verify() (err error) {
 // to Vault with the credentials in config and gets a client
 // token for future api calls.
 func NewVault(config VaultConfig) (KMS, error) {
-	if config.IsEmpty() {
-		return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
+	if !config.Enabled {
+		return nil, nil
 	}
 	if err := config.Verify(); err != nil {
 		return nil, err

cmd/crypto/vault_test.go

@@ -15,7 +15,6 @@
 package crypto
 
 import (
-	"fmt"
 	"testing"
 )
 
@@ -23,17 +22,17 @@ var verifyVaultConfigTests = []struct {
 	Config     VaultConfig
 	ShouldFail bool
 }{
-	{
-		ShouldFail: false, // 0
-		Config:     emptyVaultConfig,
-	},
 	{
 		ShouldFail: true,
-		Config:     VaultConfig{Endpoint: "https://127.0.0.1:8080"},
+		Config: VaultConfig{
+			Endpoint: "https://127.0.0.1:8080",
+			Enabled:  true,
+		},
 	},
 	{
 		ShouldFail: true, // 1
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth:     VaultAuth{Type: "unsupported"},
 		},
@@ -41,6 +40,7 @@ var verifyVaultConfigTests = []struct {
 	{
 		ShouldFail: true, // 2
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth: VaultAuth{
 				Type:    "approle",
@@ -51,6 +51,7 @@ var verifyVaultConfigTests = []struct {
 	{
 		ShouldFail: true, // 3
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth: VaultAuth{
 				Type:    "approle",
@@ -61,6 +62,7 @@ var verifyVaultConfigTests = []struct {
 	{
 		ShouldFail: true, // 4
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth: VaultAuth{
 				Type:    "approle",
@@ -71,6 +73,7 @@ var verifyVaultConfigTests = []struct {
 	{
 		ShouldFail: true, // 5
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth: VaultAuth{
 				Type:    "approle",
@@ -82,9 +85,9 @@ var verifyVaultConfigTests = []struct {
 }
 
 func TestVerifyVaultConfig(t *testing.T) {
-	for i, test := range verifyVaultConfigTests {
+	for _, test := range verifyVaultConfigTests {
 		test := test
-		t.Run(fmt.Sprintf("Test-%d", i), func(t *testing.T) {
+		t.Run(test.Config.Endpoint, func(t *testing.T) {
 			err := test.Config.Verify()
 			if test.ShouldFail && err == nil {
 				t.Errorf("Verify should fail but returned 'err == nil'")