Migrate minio etcd config to backend config (#7751)

etcd when used in federated setups, currently
mandates that all clusters should have same
config.json, which is too restrictive and makes
federation a restrictive environment.

This change makes it apparent that each cluster
needs to be independently managed if necessary
from `mc admin info` command line.

Each cluster with in federation can have their
own root credentials and as well as separate
regions. This way buckets get further restrictions
and allows for root creds to be not common
across clusters/data centers.

Existing data in etcd gets migrated to backend
on each clusters, upon start. Once done
users can change their config entries
independently.

cmd/config-common.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/logger"
@@ -53,8 +52,19 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 }
 
 func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
-	_, err := client.Delete(ctx, configFile)
-	return err
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+
+	_, err := client.Delete(timeoutCtx, configFile)
+	if err != nil {
+		if err == context.DeadlineExceeded {
+			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
+				client.Endpoints())
+		}
+		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
+			err, client.Endpoints())
+	}
+	return nil
 }
 
 func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
@@ -89,9 +99,11 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	resp, err := client.Get(timeoutCtx, configFile)
 	if err != nil {
 		if err == context.DeadlineExceeded {
-			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
+				client.Endpoints())
 		}
-		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
+		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
+			err, client.Endpoints())
 	}
 	if resp.Count == 0 {
 		return nil, errConfigNotFound
@@ -104,54 +116,7 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	return nil, errConfigNotFound
 }
 
-// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
-func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
-	for {
-		watchCh := globalEtcdClient.Watch(context.Background(), configFile)
-		select {
-		case <-GlobalServiceDoneCh:
-			return
-		case watchResp, ok := <-watchCh:
-			if !ok {
-				time.Sleep(1 * time.Second)
-				continue
-			}
-			if err := watchResp.Err(); err != nil {
-				logger.LogIf(context.Background(), err)
-				// log and retry.
-				time.Sleep(1 * time.Second)
-				continue
-			}
-			for _, event := range watchResp.Events {
-				if event.IsModify() || event.IsCreate() {
-					loadCfgFn(objAPI)
-				}
-			}
-		}
-	}
-}
-
-func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
-	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
-	defer cancel()
-	resp, err := client.Get(timeoutCtx, configFile)
-	if err != nil {
-		if err == context.DeadlineExceeded {
-			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
-		}
-		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
-	}
-	if resp.Count == 0 {
-		return errConfigNotFound
-	}
-	return nil
-}
-
 func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
-	if globalEtcdClient != nil {
-		return checkConfigEtcd(ctx, globalEtcdClient, configFile)
-	}
-
 	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {

cmd/config-migrate.go

@@ -29,7 +29,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
@@ -2413,6 +2413,7 @@ func migrateV27ToV28() error {
 }
 
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
+// if etcd is configured then migrates /config/config.json to '<export_path>/.minio.sys/config/config.json'
 func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
@@ -2423,10 +2424,14 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	}
 
 	defer func() {
-		// Rename config.json to config.json.deprecated only upon
-		// success of this function.
 		if err == nil {
-			os.Rename(getConfigFile(), getConfigFile()+".deprecated")
+			if globalEtcdClient != nil {
+				deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+			} else {
+				// Rename config.json to config.json.deprecated only upon
+				// success of this function.
+				os.Rename(getConfigFile(), getConfigFile()+".deprecated")
+			}
 		}
 	}()
 
@@ -2446,20 +2451,24 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
+	var configFiles = []string{
+		getConfigFile(),
+		getConfigFile() + ".deprecated",
+		configFile,
+	}
 	var config = &serverConfig{}
-	if _, err = Load(getConfigFile(), config); err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-		// Read from deprecate file as well if necessary.
-		if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
+	for _, cfgFile := range configFiles {
+		if _, err = Load(cfgFile, config); err != nil {
 			if !os.IsNotExist(err) {
 				return err
 			}
-			// If all else fails simply initialize the server config.
-			return newSrvConfig(objAPI)
+			continue
 		}
-
+		break
+	}
+	if os.IsNotExist(err) {
+		// Initialize the server config, if no config exists.
+		return newSrvConfig(objAPI)
 	}
 	return saveServerConfig(context.Background(), objAPI, config)
 }

cmd/config.go

@@ -49,10 +49,6 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 	}
 
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		return saveConfigEtcd(ctx, globalEtcdClient, configFile, data)
-	}
-
 	// Create a backup of the current config
 	oldData, err := readConfig(ctx, objAPI, configFile)
 	if err == nil {
@@ -71,15 +67,8 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 }
 
 func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
-	var configData []byte
-	var err error
-
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
-	} else {
-		configData, err = readConfig(ctx, objAPI, configFile)
-	}
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
 		return nil, err
 	}
@@ -152,45 +141,25 @@ func initConfig(objAPI ObjectLayer) error {
 		return errServerNotInitialized
 	}
 
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	if globalEtcdClient != nil {
-		if err := checkConfigEtcd(context.Background(), globalEtcdClient, getConfigFile()); err != nil {
-			if err == errConfigNotFound {
-				// Migrates all configs at old location.
-				if err = migrateConfig(); err != nil {
-					return err
-				}
-				// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
-				if err = migrateConfigToMinioSys(objAPI); err != nil {
-					return err
-				}
-			} else {
-				return err
-			}
-		}
-
-		// Watch config for changes and reloads them.
-		go watchConfigEtcd(objAPI, configFile, loadConfig)
-
-	} else {
-		if isFile(getConfigFile()) {
-			if err := migrateConfig(); err != nil {
-				return err
-			}
-		}
-		// Migrates ${HOME}/.minio/config.json or config.json.deprecated
-		// to '<export_path>/.minio.sys/config/config.json'
-		// ignore if the file doesn't exist.
-		if err := migrateConfigToMinioSys(objAPI); err != nil {
-			return err
-		}
-
-		// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
-		if err := migrateMinioSysConfig(objAPI); err != nil {
+	if isFile(getConfigFile()) {
+		if err := migrateConfig(); err != nil {
 			return err
 		}
 	}
 
+	// Migrates ${HOME}/.minio/config.json or config.json.deprecated
+	// to '<export_path>/.minio.sys/config/config.json'
+	// ignore if the file doesn't exist.
+	// If etcd is set then migrates /config/config.json
+	// to '<export_path>/.minio.sys/config/config.json'
+	if err := migrateConfigToMinioSys(objAPI); err != nil {
+		return err
+	}
+
+	// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
+	if err := migrateMinioSysConfig(objAPI); err != nil {
+		return err
+	}
+
 	return loadConfig(objAPI)
 }

docs/federation/lookup/README.md

@@ -9,8 +9,8 @@ Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quicksta
 ### 2. Run MinIO in federated mode
 Bucket lookup from DNS federation requires two dependencies
 
-- etcd (for config, bucket SRV records)
-- CoreDNS (for DNS management based on populated bucket SRV records, optional)
+- etcd (for bucket DNS service records)
+- CoreDNS (for DNS management based on populated bucket DNS service records, optional)
 
 ## Architecture