From 477230c82e3c8f07c9c0ef21107d33240d7ef1cb Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 21 Apr 2023 13:56:08 -0700 Subject: [PATCH] avoid attempting to migrate old configs (#17004) --- cmd/config-encrypted.go | 215 ----------------------- cmd/config-migrate.go | 341 ++++++++----------------------------- cmd/config-migrate_test.go | 5 +- cmd/config-versions.go | 101 ----------- cmd/config.go | 83 ++++----- cmd/iam-etcd-store.go | 21 --- cmd/iam-object-store.go | 38 ++++- cmd/iam.go | 15 -- cmd/server-main.go | 63 ++----- 9 files changed, 143 insertions(+), 739 deletions(-) delete mode 100644 cmd/config-encrypted.go diff --git a/cmd/config-encrypted.go b/cmd/config-encrypted.go deleted file mode 100644 index d4a1d7a2f..000000000 --- a/cmd/config-encrypted.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright (c) 2015-2021 MinIO, Inc. -// -// This file is part of MinIO Object Storage stack -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package cmd - -import ( - "bytes" - "context" - "fmt" - "path" - "time" - "unicode/utf8" - - "github.com/minio/madmin-go/v2" - "github.com/minio/minio/internal/config" - "github.com/minio/minio/internal/kms" - "github.com/minio/minio/internal/logger" - etcd "go.etcd.io/etcd/client/v3" -) - -func handleEncryptedConfigBackend(objAPI ObjectLayer) error { - encrypted, err := checkBackendEncrypted(objAPI) - if err != nil { - return fmt.Errorf("Unable to encrypt config %w", err) - } - if !encrypted { - return nil - } - if err = migrateConfigPrefixToEncrypted(objAPI); err != nil { - return fmt.Errorf("Unable to migrate all config at .minio.sys/config/: %w", err) - } - return nil -} - -const backendEncryptedFile = "backend-encrypted" - -var backendEncryptedMigrationComplete = []byte("encrypted") - -func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool, error) { - bootstrapTrace("check if etcd backend is encrypted") - data, err := readKeyEtcd(ctx, client, backendEncryptedFile) - if err != nil && err != errConfigNotFound { - return false, err - } - return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil -} - -func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) { - bootstrapTrace("check if the config backend is encrypted") - data, err := readConfig(GlobalContext, objAPI, backendEncryptedFile) - if err != nil && err != errConfigNotFound { - return false, err - } - return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil -} - -func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error { - encrypted, err := checkBackendEtcdEncrypted(ctx, client) - if err != nil { - return err - } - - // If backend doesn't have this file means we have already - // attempted then migration - if !encrypted { - return nil - } - - bootstrapTrace("encrypt etcd config") - - if GlobalKMS != nil { - stat, err := GlobalKMS.Stat(ctx) - if err != nil { - return err - } - logger.Info(fmt.Sprintf("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name)) - } - - listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - - r, err := client.Get(listCtx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly()) - if err != nil { - return err - } - - for _, kv := range r.Kvs { - data, err := readKeyEtcd(ctx, client, string(kv.Key)) - if err == errConfigNotFound { // Perhaps not present or someone deleted it. - continue - } - if err != nil { - return err - } - - if !utf8.Valid(data) { - pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data)) - if err != nil { - if GlobalKMS != nil { - pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)), - }) - if err != nil { - pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: string(kv.Key), - }) - if err != nil { - return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err) - } - } - } else { - return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err) - } - } - data = pdata - } - - if GlobalKMS != nil { - data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)), - }) - if err != nil { - return err - } - } - - if err = saveKeyEtcd(ctx, client, string(kv.Key), data); err != nil { - return err - } - } - - if GlobalKMS != nil { - logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.") - } - - return deleteKeyEtcd(ctx, client, backendEncryptedFile) -} - -func migrateConfigPrefixToEncrypted(objAPI ObjectLayer) error { - bootstrapTrace("migrating config prefix to encrypted") - if GlobalKMS != nil { - stat, err := GlobalKMS.Stat(context.Background()) - if err != nil { - return err - } - logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name)) - } - - results := make(chan ObjectInfo) - if err := objAPI.Walk(GlobalContext, minioMetaBucket, minioConfigPrefix, results, ObjectOptions{}); err != nil { - return err - } - - for obj := range results { - data, err := readConfig(GlobalContext, objAPI, obj.Name) - if err != nil { - return err - } - - if !utf8.Valid(data) { - pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data)) - if err != nil { - if GlobalKMS != nil { - pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: path.Join(minioMetaBucket, obj.Name), - }) - if err != nil { - pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: obj.Name, - }) - if err != nil { - return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err) - } - } - } else { - return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err) - } - } - data = pdata - } - - if GlobalKMS != nil { - data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{ - obj.Bucket: path.Join(obj.Bucket, obj.Name), - }) - if err != nil { - return err - } - } - - if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil { - return err - } - } - - if GlobalKMS != nil { - logger.Info("Migration of encrypted config data completed. All config data is now encrypted.") - } - - return deleteConfig(GlobalContext, objAPI, backendEncryptedFile) -} diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index 2e9e1593e..d52741db0 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -18,16 +18,15 @@ package cmd import ( - "bytes" + "context" "encoding/json" + "errors" "fmt" "os" "path" "path/filepath" "strings" - "unicode/utf8" - "github.com/minio/madmin-go/v2" "github.com/minio/minio/internal/auth" "github.com/minio/minio/internal/config" "github.com/minio/minio/internal/config/cache" @@ -39,7 +38,6 @@ import ( "github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/event" "github.com/minio/minio/internal/event/target" - "github.com/minio/minio/internal/kms" "github.com/minio/minio/internal/logger" "github.com/minio/minio/internal/logger/target/http" xnet "github.com/minio/pkg/net" @@ -2471,284 +2469,91 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) { return saveServerConfig(GlobalContext, objAPI, config) } -// Migrates '.minio.sys/config.json' to v33. -func migrateMinioSysConfig(objAPI ObjectLayer) error { - bootstrapTrace("migrate .minio.sys/config/config.json to latest version") +func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.Config, error) { // Construct path to config.json for the given bucket. configFile := path.Join(minioConfigPrefix, minioConfigFile) - // Check if the config version is latest, if not migrate. - ok, _, err := checkConfigVersion(objAPI, configFile, "33") - if err != nil { - return err - } - if ok { - return nil + configFiles := []string{ + getConfigFile(), + getConfigFile() + ".deprecated", + configFile, } - if err := migrateV27ToV28MinioSys(objAPI); err != nil { - return err - } - if err := migrateV28ToV29MinioSys(objAPI); err != nil { - return err - } - if err := migrateV29ToV30MinioSys(objAPI); err != nil { - return err - } - if err := migrateV30ToV31MinioSys(objAPI); err != nil { - return err - } - if err := migrateV31ToV32MinioSys(objAPI); err != nil { - return err - } - return migrateV32ToV33MinioSys(objAPI) -} + newServerCfg := func() (config.Config, error) { + // Initialize server config. + srvCfg := newServerConfig() -func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) { - data, err := readConfig(GlobalContext, objAPI, configFile) - if err != nil { - return false, nil, err + return srvCfg, saveServerConfig(ctx, objAPI, srvCfg) } - if !utf8.Valid(data) { - if GlobalKMS != nil { - data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: path.Join(minioMetaBucket, configFile), - }) - } + var data []byte + var err error - if GlobalKMS == nil && err != nil { - data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data)) - if err != nil { - if err == madmin.ErrMaliciousData || err == madmin.ErrUnexpectedHeader { - return false, nil, config.ErrInvalidConfigDecryptionKey(nil) - } - return false, nil, err + cfg := &serverConfigV33{} + for _, cfgFile := range configFiles { + if _, err = Load(cfgFile, cfg); err != nil { + if !osIsNotExist(err) && !osIsPermission(err) { + return nil, err } + continue + } + data, _ = json.Marshal(cfg) + break + } + if osIsPermission(err) { + logger.Info("Older config found but is not readable %s, proceeding to read config from other places", err) + } + if osIsNotExist(err) || osIsPermission(err) || len(data) == 0 { + data, err = readConfig(GlobalContext, objAPI, configFile) + if err != nil { + // when config.json is not found, then we freshly initialize. + if errors.Is(err, errConfigNotFound) { + return newServerCfg() + } + return nil, err + } + + data, err = decryptData(data, configFile) + if err != nil { + return nil, err + } + + newCfg, err := readServerConfig(GlobalContext, objAPI, data) + if err == nil { + return newCfg, nil + } + + // Read older `.minio.sys/config/config.json`, if not + // possible just fail. + if err = json.Unmarshal(data, cfg); err != nil { + // Unable to parse old JSON simply re-initialize a new one. + return newServerCfg() } } - if version == "kvs" { - // Check api config values are present. - var vcfg struct { - API struct { - E []struct { - Key string `json:"key"` - Value string `json:"value"` - } `json:"_"` - } `json:"api"` + // Init compression config. For future migration, Compression config needs to be copied over from previous version. + switch cfg.Version { + case "29": + // V29 -> V30 + cfg.Compression.Enabled = false + cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator) + cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator) + case "30": + // V30 -> V31 + cfg.OpenID = openid.Config{} + cfg.Policy.OPA = opa.Args{ + URL: &xnet.URL{}, + AuthToken: "", } - if err = json.Unmarshal(data, &vcfg); err != nil { - return false, nil, nil - } - return len(vcfg.API.E) > 0, data, nil - } - var versionConfig struct { - Version string `json:"version"` - } - - vcfg := &versionConfig - if err = json.Unmarshal(data, vcfg); err != nil { - return false, nil, err - } - return vcfg.Version == version, data, nil -} - -func migrateV27ToV28MinioSys(objAPI ObjectLayer) error { - configFile := path.Join(minioConfigPrefix, minioConfigFile) - ok, data, err := checkConfigVersion(objAPI, configFile, "27") - if err == errConfigNotFound { - return nil - } else if err != nil { - return fmt.Errorf("Unable to load config file. %w", err) - } - if !ok { - return nil - } - - cfg := &serverConfigV28{} - if err = json.Unmarshal(data, cfg); err != nil { - return err - } - - cfg.Version = "28" - if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil { - return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %w", err) - } - - logger.Info(configMigrateMSGTemplate, configFile, "27", "28") - return nil -} - -func migrateV28ToV29MinioSys(objAPI ObjectLayer) error { - configFile := path.Join(minioConfigPrefix, minioConfigFile) - - ok, data, err := checkConfigVersion(objAPI, configFile, "28") - if err == errConfigNotFound { - return nil - } else if err != nil { - return fmt.Errorf("Unable to load config file. %w", err) - } - if !ok { - return nil - } - - cfg := &serverConfigV29{} - if err = json.Unmarshal(data, cfg); err != nil { - return err - } - - cfg.Version = "29" - if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil { - return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %w", err) - } - - logger.Info(configMigrateMSGTemplate, configFile, "28", "29") - return nil -} - -func migrateV29ToV30MinioSys(objAPI ObjectLayer) error { - configFile := path.Join(minioConfigPrefix, minioConfigFile) - - ok, data, err := checkConfigVersion(objAPI, configFile, "29") - if err == errConfigNotFound { - return nil - } else if err != nil { - return fmt.Errorf("Unable to load config file. %w", err) - } - if !ok { - return nil - } - - cfg := &serverConfigV30{} - if err = json.Unmarshal(data, cfg); err != nil { - return err - } - - cfg.Version = "30" - // Init compression config.For future migration, Compression config needs to be copied over from previous version. - cfg.Compression.Enabled = false - cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator) - cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator) - - if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil { - return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %w", err) - } - - logger.Info(configMigrateMSGTemplate, configFile, "29", "30") - return nil -} - -func migrateV30ToV31MinioSys(objAPI ObjectLayer) error { - configFile := path.Join(minioConfigPrefix, minioConfigFile) - - ok, data, err := checkConfigVersion(objAPI, configFile, "30") - if err == errConfigNotFound { - return nil - } else if err != nil { - return fmt.Errorf("Unable to load config file. %w", err) - } - if !ok { - return nil - } - - cfg := &serverConfigV31{} - if err = json.Unmarshal(data, cfg); err != nil { - return err - } - - cfg.Version = "31" - cfg.OpenID = openid.Config{} - - cfg.Policy.OPA = opa.Args{ - URL: &xnet.URL{}, - AuthToken: "", - } - - if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil { - return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %w", err) - } - - logger.Info(configMigrateMSGTemplate, configFile, "30", "31") - return nil -} - -func migrateV31ToV32MinioSys(objAPI ObjectLayer) error { - configFile := path.Join(minioConfigPrefix, minioConfigFile) - - ok, data, err := checkConfigVersion(objAPI, configFile, "31") - if err == errConfigNotFound { - return nil - } else if err != nil { - return fmt.Errorf("Unable to load config file. %w", err) - } - if !ok { - return nil - } - - cfg := &serverConfigV32{} - if err = json.Unmarshal(data, cfg); err != nil { - return err - } - - cfg.Version = "32" - cfg.Notify.NSQ = make(map[string]target.NSQArgs) - cfg.Notify.NSQ["1"] = target.NSQArgs{} - - if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil { - return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %w", err) - } - - logger.Info(configMigrateMSGTemplate, configFile, "31", "32") - return nil -} - -func migrateV32ToV33MinioSys(objAPI ObjectLayer) error { - configFile := path.Join(minioConfigPrefix, minioConfigFile) - - ok, data, err := checkConfigVersion(objAPI, configFile, "32") - if err == errConfigNotFound { - return nil - } else if err != nil { - return fmt.Errorf("Unable to load config file. %w", err) - } - if !ok { - return nil - } - - cfg := &serverConfigV33{} - if err = json.Unmarshal(data, cfg); err != nil { - return err + case "31": + // V31 -> V32 + cfg.Notify.NSQ = make(map[string]target.NSQArgs) + cfg.Notify.NSQ["1"] = target.NSQArgs{} } + // Move to latest. cfg.Version = "33" - if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil { - return fmt.Errorf("Failed to migrate config from '32' to '33' . %w", err) - } - - logger.Info(configMigrateMSGTemplate, configFile, "32", "33") - return nil -} - -func migrateMinioSysConfigToKV(objAPI ObjectLayer) error { - bootstrapTrace("migrate config to KV style") - configFile := path.Join(minioConfigPrefix, minioConfigFile) - - // Check if the config version is latest, if not migrate. - ok, data, err := checkConfigVersion(objAPI, configFile, "33") - if err != nil { - return err - } - if !ok { - return nil - } - - cfg := &serverConfigV33{} - if err = json.Unmarshal(data, cfg); err != nil { - return err - } - newCfg := newServerConfig() config.SetRegion(newCfg, cfg.Region) @@ -2797,11 +2602,5 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error { notify.SetNotifyWebhook(newCfg, k, args) } - if err = saveServerConfig(GlobalContext, objAPI, newCfg); err != nil { - return err - } - - logger.Info("Configuration file %s migrated from version '%s' to new KV format successfully.", - configFile, "33") - return nil + return newCfg, nil } diff --git a/cmd/config-migrate_test.go b/cmd/config-migrate_test.go index 6a25cd290..4e4d4e70a 100644 --- a/cmd/config-migrate_test.go +++ b/cmd/config-migrate_test.go @@ -196,11 +196,12 @@ func TestServerConfigMigrateV2toV33(t *testing.T) { t.Fatal("Unexpected error: ", err) } - if err := migrateMinioSysConfig(objLayer); err != nil { + srvCfg, err := readConfigWithoutMigrate(context.Background(), objLayer) + if err != nil { t.Fatal("Unexpected error: ", err) } - if err := migrateMinioSysConfigToKV(objLayer); err != nil { + if err = saveServerConfig(GlobalContext, objLayer, srvCfg); err != nil { t.Fatal("Unexpected error: ", err) } diff --git a/cmd/config-versions.go b/cmd/config-versions.go index 5176adcc3..f359f8338 100644 --- a/cmd/config-versions.go +++ b/cmd/config-versions.go @@ -717,107 +717,6 @@ type serverConfigV28 struct { Logger logger.Config `json:"logger"` } -// serverConfigV29 is just like version '28'. -type serverConfigV29 serverConfigV28 - -// serverConfigV30 is just like version '29', stores additionally -// extensions and mimetypes fields for compression. -type serverConfigV30 struct { - Version string `json:"version"` - - // S3 API configuration. - Credential auth.Credentials `json:"credential"` - Region string `json:"region"` - Worm config.BoolFlag `json:"worm"` - - // Storage class configuration - StorageClass storageclass.Config `json:"storageclass"` - - // Cache configuration - Cache cache.Config `json:"cache"` - - // Notification queue configuration. - Notify notifierV3 `json:"notify"` - - // Logger configuration - Logger logger.Config `json:"logger"` - - // Compression configuration - Compression compress.Config `json:"compress"` -} - -// serverConfigV31 is just like version '30', with OPA and OpenID configuration. -type serverConfigV31 struct { - Version string `json:"version"` - - // S3 API configuration. - Credential auth.Credentials `json:"credential"` - Region string `json:"region"` - Worm config.BoolFlag `json:"worm"` - - // Storage class configuration - StorageClass storageclass.Config `json:"storageclass"` - - // Cache configuration - Cache cache.Config `json:"cache"` - - // Notification queue configuration. - Notify notifierV3 `json:"notify"` - - // Logger configuration - Logger logger.Config `json:"logger"` - - // Compression configuration - Compression compress.Config `json:"compress"` - - // OpenID configuration - OpenID openid.Config `json:"openid"` - - // External policy enforcements. - Policy struct { - // OPA configuration. - OPA opa.Args `json:"opa"` - - // Add new external policy enforcements here. - } `json:"policy"` -} - -// serverConfigV32 is just like version '31' with added nsq notifer. -type serverConfigV32 struct { - Version string `json:"version"` - - // S3 API configuration. - Credential auth.Credentials `json:"credential"` - Region string `json:"region"` - Worm config.BoolFlag `json:"worm"` - - // Storage class configuration - StorageClass storageclass.Config `json:"storageclass"` - - // Cache configuration - Cache cache.Config `json:"cache"` - - // Notification queue configuration. - Notify notify.Config `json:"notify"` - - // Logger configuration - Logger logger.Config `json:"logger"` - - // Compression configuration - Compression compress.Config `json:"compress"` - - // OpenID configuration - OpenID openid.Config `json:"openid"` - - // External policy enforcements. - Policy struct { - // OPA configuration. - OPA opa.Args `json:"opa"` - - // Add new external policy enforcements here. - } `json:"policy"` -} - // serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets. type serverConfigV33 struct { quick.Config `json:"-"` // ignore interfaces diff --git a/cmd/config.go b/cmd/config.go index 63091c44c..1ce4aba8e 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -25,7 +25,6 @@ import ( "path" "sort" "strings" - "unicode/utf8" jsoniter "github.com/json-iterator/go" "github.com/minio/madmin-go/v2" @@ -65,16 +64,16 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b if withData { data, err := readConfig(ctx, objAPI, obj.Name) if err != nil { - return nil, err + // ignore history file if not readable. + continue } - if GlobalKMS != nil { - data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - obj.Bucket: path.Join(obj.Bucket, obj.Name), - }) - if err != nil { - return nil, err - } + + data, err = decryptData(data, obj.Name) + if err != nil { + // ignore history file that cannot be loaded. + continue } + cfgEntry.Data = string(data) } configHistory = append(configHistory, cfgEntry) @@ -110,12 +109,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str return nil, err } - if GlobalKMS != nil { - data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: path.Join(minioMetaBucket, historyFile), - }) - } - return data, err + return decryptData(data, historyFile) } func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error { @@ -167,14 +161,10 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (con return nil, err } - if GlobalKMS != nil && !utf8.Valid(data) { - data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: path.Join(minioMetaBucket, configFile), - }) - if err != nil { - lookupConfigs(srvCfg, objAPI) - return nil, err - } + data, err = decryptData(data, configFile) + if err != nil { + lookupConfigs(srvCfg, objAPI) + return nil, err } } @@ -205,47 +195,32 @@ func NewConfigSys() *ConfigSys { } // Initialize and load config from remote etcd or local config directory -func initConfig(objAPI ObjectLayer) error { +func initConfig(objAPI ObjectLayer) (err error) { bootstrapTrace("load the configuration") + defer func() { + if err != nil { + bootstrapTrace(fmt.Sprintf("loading configuration failed: %v", err)) + } + }() if objAPI == nil { return errServerNotInitialized } - if isFile(getConfigFile()) { - if err := migrateConfig(); err != nil { - return err - } - } - - // Check if the config version is latest (kvs), if not migrate. - ok, data, err := checkConfigVersion(objAPI, path.Join(minioConfigPrefix, minioConfigFile), "kvs") - if err != nil && !errors.Is(err, errConfigNotFound) { + srvCfg, err := readConfigWithoutMigrate(GlobalContext, objAPI) + if err != nil { return err } - if ok { - return loadConfig(objAPI, data) - } - // Migrates ${HOME}/.minio/config.json or config.json.deprecated - // to '/.minio.sys/config/config.json' - // ignore if the file doesn't exist. - // If etcd is set then migrates /config/config.json - // to '/.minio.sys/config/config.json' - if err := migrateConfigToMinioSys(objAPI); err != nil { - return fmt.Errorf("migrateConfigToMinioSys: %w", err) - } + bootstrapTrace("lookup the configuration") - // Migrates backend '/.minio.sys/config/config.json' to latest version. - if err := migrateMinioSysConfig(objAPI); err != nil { - return fmt.Errorf("migrateMinioSysConfig: %w", err) - } + // Override any values from ENVs. + lookupConfigs(srvCfg, objAPI) - // Migrates backend '/.minio.sys/config/config.json' to - // latest config format. - if err := migrateMinioSysConfigToKV(objAPI); err != nil { - return fmt.Errorf("migrateMinioSysConfigToKV: %w", err) - } + // hold the mutex lock before a new config is assigned. + globalServerConfigMu.Lock() + globalServerConfig = srvCfg + globalServerConfigMu.Unlock() - return loadConfig(objAPI, nil) + return nil } diff --git a/cmd/iam-etcd-store.go b/cmd/iam-etcd-store.go index e688f7c36..6f84e69b2 100644 --- a/cmd/iam-etcd-store.go +++ b/cmd/iam-etcd-store.go @@ -25,7 +25,6 @@ import ( "strings" "sync" "time" - "unicode/utf8" jsoniter "github.com/json-iterator/go" "github.com/minio/minio-go/v7/pkg/set" @@ -115,26 +114,6 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item interface{}, it return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...) } -func decryptData(data []byte, itemPath string) ([]byte, error) { - var err error - if !utf8.Valid(data) && GlobalKMS != nil { - data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: path.Join(minioMetaBucket, itemPath), - }) - if err != nil { - // This fallback is needed because of a bug, in kms.Context{} - // construction during migration. - data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: itemPath, - }) - if err != nil { - return nil, err - } - } - } - return data, nil -} - func getIAMConfig(item interface{}, data []byte, itemPath string) error { data, err := decryptData(data, itemPath) if err != nil { diff --git a/cmd/iam-object-store.go b/cmd/iam-object-store.go index fc2ad9b40..8f569d398 100644 --- a/cmd/iam-object-store.go +++ b/cmd/iam-object-store.go @@ -18,6 +18,7 @@ package cmd import ( + "bytes" "context" "errors" "fmt" @@ -27,6 +28,7 @@ import ( "unicode/utf8" jsoniter "github.com/json-iterator/go" + "github.com/minio/madmin-go/v2" "github.com/minio/minio/internal/config" "github.com/minio/minio/internal/kms" "github.com/minio/minio/internal/logger" @@ -91,18 +93,40 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item interface{} return saveConfig(ctx, iamOS.objAPI, objPath, data) } +func decryptData(data []byte, objPath string) ([]byte, error) { + if utf8.Valid(data) { + return data, nil + } + + pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data)) + if err == nil { + return pdata, nil + } + if GlobalKMS != nil { + pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ + minioMetaBucket: path.Join(minioMetaBucket, objPath), + }) + if err == nil { + return pdata, nil + } + pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ + minioMetaBucket: objPath, + }) + if err == nil { + return pdata, nil + } + } + return nil, err +} + func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context, objPath string) ([]byte, ObjectInfo, error) { data, meta, err := readConfigWithMetadata(ctx, iamOS.objAPI, objPath, ObjectOptions{}) if err != nil { return nil, meta, err } - if !utf8.Valid(data) && GlobalKMS != nil { - data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ - minioMetaBucket: path.Join(minioMetaBucket, objPath), - }) - if err != nil { - return nil, meta, err - } + data, err = decryptData(data, objPath) + if err != nil { + return nil, meta, err } return data, meta, nil } diff --git a/cmd/iam.go b/cmd/iam.go index a033ec966..0f0450f08 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -286,21 +286,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc // Migrate storage format if needed. for { - if etcdClient != nil { - // **** WARNING **** - // Migrating to encrypted backend on etcd should happen before initialization of - // IAM sub-system, make sure that we do not move the above codeblock elsewhere. - if err := migrateIAMConfigsEtcdToEncrypted(retryCtx, etcdClient); err != nil { - if errors.Is(err, errEtcdUnreachable) { - logger.Info("Connection to etcd timed out. Retrying..") - continue - } - logger.LogIf(ctx, fmt.Errorf("Unable to decrypt an encrypted ETCD backend for IAM users and policies: %w", err)) - logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, some users may not be available")) - return - } - } - // Migrate IAM configuration, if necessary. if err := saveIAMFormat(retryCtx, sys.store); err != nil { if configRetriableErrors(err) { diff --git a/cmd/server-main.go b/cmd/server-main.go index 27fb90f6b..34971f21b 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -382,17 +382,8 @@ func initServer(ctx context.Context, newObject ObjectLayer) error { // Once the config is fully loaded, initialize the new object layer. setObjectLayer(newObject) - // **** WARNING **** - // Migrating to encrypted backend should happen before initialization of any - // sub-systems, make sure that we do not move the above codeblock elsewhere. - r := rand.New(rand.NewSource(time.Now().UnixNano())) - lockTimeout := newDynamicTimeoutWithOpts(dynamicTimeoutOpts{ - timeout: 5 * time.Second, - minimum: 3 * time.Second, - }) - for { select { case <-ctx.Done(): @@ -401,57 +392,23 @@ func initServer(ctx context.Context, newObject ObjectLayer) error { default: } - bootstrapTrace("trying to acquire transaction.lock") - - // Make sure to hold lock for entire migration to avoid - // such that only one server should migrate the entire config - // at a given time, this big transaction lock ensures this - // appropriately. This is also true for rotation of encrypted - // content. - txnLk := newObject.NewNSLock(minioMetaBucket, minioConfigPrefix+"/transaction.lock") - - // let one of the server acquire the lock, if not let them timeout. - // which shall be retried again by this loop. - lkctx, err := txnLk.GetLock(ctx, lockTimeout) - if err != nil { - logger.Info("Waiting for all MinIO sub-systems to be initialized.. trying to acquire lock") - waitDuration := time.Duration(r.Float64() * 5 * float64(time.Second)) - bootstrapTrace(fmt.Sprintf("lock not available. error: %v. sleeping for %v before retry", err, waitDuration)) - - // Sleep 0 -> 5 seconds, provider a higher range such that sleeps() - // and retries for lock are more spread out, needed orchestrated - // systems take 30s minimum to respond to DNS resolvers. - // - // Do not change this value. - time.Sleep(waitDuration) - continue - } - // These messages only meant primarily for distributed setup, so only log during distributed setup. if globalIsDistErasure { - logger.Info("Waiting for all MinIO sub-systems to be initialized.. lock acquired") + logger.Info("Waiting for all MinIO sub-systems to be initialize...") } - // Migrate all backend configs to encrypted backend configs, optionally - // handles rotating keys for encryption, if there is any retriable failure - // that shall be retried if there is an error. - if err = handleEncryptedConfigBackend(newObject); err == nil { - // Upon success migrating the config, initialize all sub-systems - // if all sub-systems initialized successfully return right away - if err = initConfigSubsystem(lkctx.Context(), newObject); err == nil { - txnLk.Unlock(lkctx) - // All successful return. - if globalIsDistErasure { - // These messages only meant primarily for distributed setup, so only log during distributed setup. - logger.Info("All MinIO sub-systems initialized successfully in %s", time.Since(t1)) - } - return nil + // Upon success migrating the config, initialize all sub-systems + // if all sub-systems initialized successfully return right away + err := initConfigSubsystem(ctx, newObject) + if err == nil { + // All successful return. + if globalIsDistErasure { + // These messages only meant primarily for distributed setup, so only log during distributed setup. + logger.Info("All MinIO sub-systems initialized successfully in %s", time.Since(t1)) } + return nil } - // Unlock the transaction lock and allow other nodes to acquire the lock if possible. - txnLk.Unlock(lkctx) - if configRetriableErrors(err) { logger.Info("Waiting for all MinIO sub-systems to be initialized.. possible cause (%v)", err) time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))