Add functionality to add old buckets to etcd on startup

Buckets already present on a Minio server before it joins a
bucket federated deployment will now be added to etcd during
startup. In case of a bucket name collision, admin is informed
via Minio server console message.

Added configuration migration for configuration stored in etcd
backend.

Also, environment variables are updated and ListBucket path style
request is no longer forwarded.

cmd/bucket-handlers.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -34,8 +36,52 @@ import (
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+// Check if there are buckets on server without corresponding entry in etcd backend and
+// make entries. Here is the general flow
+// - Range over all the available buckets
+// - Check if a bucket has an entry in etcd backend
+// -- If no, make an entry
+// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
+// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
+func initFederatorBackend(objLayer ObjectLayer) {
+	// List all buckets
+	b, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return
+	}
+
+	g := errgroup.WithNErrs(len(b))
+	for index := range b {
+		index := index
+		g.Go(func() error {
+			r, gerr := globalDNSConfig.Get(b[index].Name)
+			if gerr != nil {
+				if client.IsKeyNotFound(gerr) {
+					// Make a new entry
+					return globalDNSConfig.Put(b[index].Name)
+				}
+				return gerr
+			}
+			if r.Host != globalDomainIP {
+				// Log error that entry already present for different host
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use %s to access the bucket, or rename it to a unique value", b[index].Name, globalDomainIP)
+			}
+			return nil
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return
+		}
+	}
+}
+
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
@@ -158,7 +204,6 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
-
 	// If etcd, dns federation configured list buckets from etcd.
 	var bucketsInfo []BucketInfo
 	if globalDNSConfig != nil {

cmd/common-main.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"net"
 	"net/http"
@@ -45,9 +46,17 @@ func checkUpdate(mode string) {
 	}
 }
 
+// Initialize and load config from remote etcd or local config directory
 func initConfig() {
 	if globalEtcdClient != nil {
-		if err := loadConfig(); err != nil {
+		kapi := etcdc.NewKeysAPI(globalEtcdClient)
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		_, err := kapi.Get(ctx, getConfigFile(), nil)
+		cancel()
+		if err == nil {
+			logger.FatalIf(migrateConfig(), "Config migration failed.")
+			logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+		} else {
 			if etcdc.IsKeyNotFound(err) {
 				logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
 				logger.Info("Created minio configuration file successfully at", globalEtcdClient.Endpoints())
@@ -58,11 +67,11 @@ func initConfig() {
 		return
 	}
 
-	// Config file does not exist, we create it fresh and return upon success.
 	if isFile(getConfigFile()) {
 		logger.FatalIf(migrateConfig(), "Config migration failed")
 		logger.FatalIf(loadConfig(), "Unable to load the configuration file")
 	} else {
+		// Config file does not exist, we create it fresh and return upon success.
 		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
 		logger.Info("Created minio configuration file successfully at " + getConfigDir())
 	}
@@ -159,7 +168,7 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
 	}
 
-	globalDomainIP = os.Getenv("MINIO_DOMAIN_IP")
+	globalDomainIP = os.Getenv("MINIO_PUBLIC_IP")
 	if globalDomainName != "" && globalDomainIP != "" && globalEtcdClient != nil {
 		var err error
 		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIP, globalMinioPort, globalEtcdClient)

cmd/config-current.go

@@ -127,10 +127,28 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 	return s.Cache
 }
 
-// Save config.
+// Save config file to corresponding backend
-func (s *serverConfig) Save(configFile string) error {
+func Save(configFile string, data interface{}) error {
-	// Save config file.
+	if globalEtcdClient == nil {
-	return quick.Save(configFile, s)
+		return quick.SaveLocalConfig(configFile, data)
+	}
+	return quick.SaveEtcdConfig(configFile, data, globalEtcdClient)
+}
+
+// Load config from backend
+func Load(configFile string, data interface{}) (quick.Config, error) {
+	if globalEtcdClient == nil {
+		return quick.LoadLocalConfig(configFile, data)
+	}
+	return quick.LoadEtcdConfig(configFile, data, globalEtcdClient)
+}
+
+// GetVersion gets config version from backend
+func GetVersion(configFile string) (string, error) {
+	if globalEtcdClient == nil {
+		return quick.GetLocalVersion(configFile)
+	}
+	return quick.GetEtcdVersion(configFile, globalEtcdClient)
 }
 
 // Returns the string describing a difference with the given
@@ -272,7 +290,7 @@ func newConfig() error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return globalServerConfig.Save(getConfigFile())
+	return Save(getConfigFile(), globalServerConfig)
 }
 
 // newQuickConfig - initialize a new server config, with an allocated

cmd/config-migrate.go

@@ -21,12 +21,12 @@ import (
 	"os"
 	"path/filepath"
 
+	etcdc "github.com/coreos/etcd/client"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	xnet "github.com/minio/minio/pkg/net"
-	"github.com/minio/minio/pkg/quick"
 )
 
 // DO NOT EDIT following message template, please open a github issue to discuss instead.
@@ -42,7 +42,7 @@ func migrateConfig() error {
 	}
 
 	// Load only config version information.
-	version, err := quick.GetVersion(getConfigFile())
+	version, err := GetVersion(getConfigFile())
 	if err != nil {
 		return err
 	}
@@ -194,8 +194,8 @@ func purgeV1() error {
 	configFile := filepath.Join(getConfigDir(), "fsUsers.json")
 
 	cv1 := &configV1{}
-	_, err := quick.Load(configFile, cv1)
+	_, err := Load(configFile, cv1)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || etcdc.IsKeyNotFound(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -215,7 +215,7 @@ func migrateV2ToV3() error {
 	configFile := getConfigFile()
 
 	cv2 := &configV2{}
-	_, err := quick.Load(configFile, cv2)
+	_, err := Load(configFile, cv2)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -259,7 +259,7 @@ func migrateV2ToV3() error {
 	}
 	srvConfig.Logger.Syslog = slogger
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv2.Version, srvConfig.Version, err)
 	}
 
@@ -274,7 +274,7 @@ func migrateV3ToV4() error {
 	configFile := getConfigFile()
 
 	cv3 := &configV3{}
-	_, err := quick.Load(configFile, cv3)
+	_, err := Load(configFile, cv3)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -297,7 +297,7 @@ func migrateV3ToV4() error {
 	srvConfig.Logger.File = cv3.Logger.File
 	srvConfig.Logger.Syslog = cv3.Logger.Syslog
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv3.Version, srvConfig.Version, err)
 	}
 
@@ -312,7 +312,7 @@ func migrateV4ToV5() error {
 	configFile := getConfigFile()
 
 	cv4 := &configV4{}
-	_, err := quick.Load(configFile, cv4)
+	_, err := Load(configFile, cv4)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -338,7 +338,7 @@ func migrateV4ToV5() error {
 	srvConfig.Logger.ElasticSearch.Enable = false
 	srvConfig.Logger.Redis.Enable = false
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv4.Version, srvConfig.Version, err)
 	}
 
@@ -353,7 +353,7 @@ func migrateV5ToV6() error {
 	configFile := getConfigFile()
 
 	cv5 := &configV5{}
-	_, err := quick.Load(configFile, cv5)
+	_, err := Load(configFile, cv5)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -427,7 +427,7 @@ func migrateV5ToV6() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv5.Version, srvConfig.Version, err)
 	}
 
@@ -442,7 +442,7 @@ func migrateV6ToV7() error {
 	configFile := getConfigFile()
 
 	cv6 := &configV6{}
-	_, err := quick.Load(configFile, cv6)
+	_, err := Load(configFile, cv6)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -483,7 +483,7 @@ func migrateV6ToV7() error {
 		srvConfig.Notify.Redis = cv6.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv6.Version, srvConfig.Version, err)
 	}
 
@@ -498,7 +498,7 @@ func migrateV7ToV8() error {
 	configFile := getConfigFile()
 
 	cv7 := &serverConfigV7{}
-	_, err := quick.Load(configFile, cv7)
+	_, err := Load(configFile, cv7)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -546,7 +546,7 @@ func migrateV7ToV8() error {
 		srvConfig.Notify.Redis = cv7.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv7.Version, srvConfig.Version, err)
 	}
 
@@ -560,7 +560,7 @@ func migrateV8ToV9() error {
 	configFile := getConfigFile()
 
 	cv8 := &serverConfigV8{}
-	_, err := quick.Load(configFile, cv8)
+	_, err := Load(configFile, cv8)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -616,7 +616,7 @@ func migrateV8ToV9() error {
 		srvConfig.Notify.PostgreSQL = cv8.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv8.Version, srvConfig.Version, err)
 	}
 
@@ -630,7 +630,7 @@ func migrateV9ToV10() error {
 	configFile := getConfigFile()
 
 	cv9 := &serverConfigV9{}
-	_, err := quick.Load(configFile, cv9)
+	_, err := Load(configFile, cv9)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -684,7 +684,7 @@ func migrateV9ToV10() error {
 		srvConfig.Notify.PostgreSQL = cv9.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv9.Version, srvConfig.Version, err)
 	}
 
@@ -698,7 +698,7 @@ func migrateV10ToV11() error {
 	configFile := getConfigFile()
 
 	cv10 := &serverConfigV10{}
-	_, err := quick.Load(configFile, cv10)
+	_, err := Load(configFile, cv10)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -755,7 +755,7 @@ func migrateV10ToV11() error {
 	srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
 	srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv10.Version, srvConfig.Version, err)
 	}
 
@@ -769,7 +769,7 @@ func migrateV11ToV12() error {
 	configFile := getConfigFile()
 
 	cv11 := &serverConfigV11{}
-	_, err := quick.Load(configFile, cv11)
+	_, err := Load(configFile, cv11)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -853,7 +853,7 @@ func migrateV11ToV12() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv11.Version, srvConfig.Version, err)
 	}
 
@@ -866,7 +866,7 @@ func migrateV12ToV13() error {
 	configFile := getConfigFile()
 
 	cv12 := &serverConfigV12{}
-	_, err := quick.Load(configFile, cv12)
+	_, err := Load(configFile, cv12)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -933,7 +933,7 @@ func migrateV12ToV13() error {
 	srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
 	srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv12.Version, srvConfig.Version, err)
 	}
 
@@ -946,7 +946,7 @@ func migrateV13ToV14() error {
 	configFile := getConfigFile()
 
 	cv13 := &serverConfigV13{}
-	_, err := quick.Load(configFile, cv13)
+	_, err := Load(configFile, cv13)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1018,7 +1018,7 @@ func migrateV13ToV14() error {
 	// Set the new browser parameter to true by default
 	srvConfig.Browser = true
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv13.Version, srvConfig.Version, err)
 	}
 
@@ -1031,7 +1031,7 @@ func migrateV14ToV15() error {
 	configFile := getConfigFile()
 
 	cv14 := &serverConfigV14{}
-	_, err := quick.Load(configFile, cv14)
+	_, err := Load(configFile, cv14)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1107,7 +1107,7 @@ func migrateV14ToV15() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv14.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv14.Version, srvConfig.Version, err)
 	}
 
@@ -1121,7 +1121,7 @@ func migrateV15ToV16() error {
 	configFile := getConfigFile()
 
 	cv15 := &serverConfigV15{}
-	_, err := quick.Load(configFile, cv15)
+	_, err := Load(configFile, cv15)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1197,7 +1197,7 @@ func migrateV15ToV16() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv15.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err)
 	}
 
@@ -1211,7 +1211,7 @@ func migrateV16ToV17() error {
 	configFile := getConfigFile()
 
 	cv16 := &serverConfigV16{}
-	_, err := quick.Load(configFile, cv16)
+	_, err := Load(configFile, cv16)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1318,7 +1318,7 @@ func migrateV16ToV17() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv16.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv16.Version, srvConfig.Version, err)
 	}
 
@@ -1332,7 +1332,7 @@ func migrateV17ToV18() error {
 	configFile := getConfigFile()
 
 	cv17 := &serverConfigV17{}
-	_, err := quick.Load(configFile, cv17)
+	_, err := Load(configFile, cv17)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1422,7 +1422,7 @@ func migrateV17ToV18() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv17.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err)
 	}
 
@@ -1434,7 +1434,7 @@ func migrateV18ToV19() error {
 	configFile := getConfigFile()
 
 	cv18 := &serverConfigV18{}
-	_, err := quick.Load(configFile, cv18)
+	_, err := Load(configFile, cv18)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1528,7 +1528,7 @@ func migrateV18ToV19() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv18.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err)
 	}
 
@@ -1540,7 +1540,7 @@ func migrateV19ToV20() error {
 	configFile := getConfigFile()
 
 	cv19 := &serverConfigV19{}
-	_, err := quick.Load(configFile, cv19)
+	_, err := Load(configFile, cv19)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1633,7 +1633,7 @@ func migrateV19ToV20() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv19.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
 	}
 
@@ -1645,7 +1645,7 @@ func migrateV20ToV21() error {
 	configFile := getConfigFile()
 
 	cv20 := &serverConfigV20{}
-	_, err := quick.Load(configFile, cv20)
+	_, err := Load(configFile, cv20)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1737,7 +1737,7 @@ func migrateV20ToV21() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv20.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
 	}
 
@@ -1749,7 +1749,7 @@ func migrateV21ToV22() error {
 	configFile := getConfigFile()
 
 	cv21 := &serverConfigV21{}
-	_, err := quick.Load(configFile, cv21)
+	_, err := Load(configFile, cv21)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1841,7 +1841,7 @@ func migrateV21ToV22() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv21.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
 	}
 
@@ -1853,7 +1853,7 @@ func migrateV22ToV23() error {
 	configFile := getConfigFile()
 
 	cv22 := &serverConfigV22{}
-	_, err := quick.Load(configFile, cv22)
+	_, err := Load(configFile, cv22)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1954,7 +1954,7 @@ func migrateV22ToV23() error {
 	srvConfig.Cache.Exclude = []string{}
 	srvConfig.Cache.Expiry = globalCacheExpiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv22.Version, srvConfig.Version, err)
 	}
 

cmd/generic-handlers.go

@@ -628,13 +628,17 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		f.handler.ServeHTTP(w, r)
 		return
 	}
-
 	bucket, object := urlPath2BucketObjectName(r.URL.Path)
+	// MakeBucket request
 	if r.Method == http.MethodPut && bucket != "" && object == "" {
 		f.handler.ServeHTTP(w, r)
 		return
 	}
-
+	// ListBucket request
+	if r.Method == http.MethodGet && bucket == "" && object == "" {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
 	sr, err := globalDNSConfig.Get(bucket)
 	if err != nil {
 		if client.IsKeyNotFound(err) {
@@ -644,7 +648,6 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		}
 		return
 	}
-
 	if sr.Host != globalDomainIP {
 		backendURL := fmt.Sprintf("http://%s:%d", sr.Host, sr.Port)
 		if globalIsSSL {
@@ -658,7 +661,6 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		f.fwd.ServeHTTP(w, r)
 		return
 	}
-
 	f.handler.ServeHTTP(w, r)
 }
 

cmd/routers.go

@@ -49,8 +49,7 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 var globalHandlers = []HandlerFunc{
 	// set HTTP security headers such as Content-Security-Policy.
 	addSecurityHeaders,
-	// set Bucket forwarding handler to proxy path style requests
+	// Forward path style requests to actual host in a bucket federated setup.
-	// when federated backend is enabled.
 	setBucketForwardingHandler,
 	// Ratelimit the incoming requests using a token bucket algorithm
 	setRateLimitHandler,

cmd/server-main.go

@@ -90,11 +90,9 @@ ENVIRONMENT VARIABLES:
      MINIO_WORM: To turn on Write-Once-Read-Many in server, set this value to "on".
 
   BUCKET-DNS:
-     MINIO_DOMAIN:    To enable virtual-host-style requests.
+     MINIO_DOMAIN:    To enable bucket DNS requests, set this value to Minio host domain name.
-     MINIO_DOMAIN_IP: To enable virtual-host-style requests.
+     MINIO_PUBLIC_IP: To enable bucket DNS requests, set this value to Minio host public IP.
-
+     MINIO_ETCD_ENDPOINTS: To enable bucket DNS requests, set this value to list of etcd endpoints delimited by ",".
-  ETCD:
-     MINIO_ETCD_ENDPOINTS: Comma separated list of etcd endpoints.
 
 EXAMPLES:
   1. Start minio server on "/home/shared" directory.
@@ -310,6 +308,11 @@ func serverMain(ctx *cli.Context) {
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()
 
+	// Populate existing buckets to the etcd backend
+	if globalDNSConfig != nil {
+		initFederatorBackend(newObject)
+	}
+
 	// Prints the formatted startup message once object layer is initialized.
 	apiEndpoints := getAPIEndpoints(globalMinioAddr)
 	printStartupMessage(apiEndpoints)

docs/federation/lookup/README.md

@@ -21,21 +21,21 @@ Bucket lookup federation requires two dependencies
 ```
 export MINIO_ETCD_ENDPOINTS="http://remote-etcd1:2379,http://remote-etcd2:4001"
 export MINIO_DOMAIN=domain.com
-export MINIO_DOMAIN_IP=44.35.2.1
+export MINIO_PUBLIC_IP=44.35.2.1
 minio server http://rack{1...4}.host{1...4}.domain.com/mnt/export{1...32}
 ```
 > cluster2
 ```
 export MINIO_ETCD_ENDPOINTS="http://remote-etcd1:2379,http://remote-etcd2:4001"
 export MINIO_DOMAIN=domain.com
-export MINIO_DOMAIN_IP=44.35.2.2
+export MINIO_PUBLIC_IP=44.35.2.2
 minio server http://rack{5...8}.host{5...8}.domain.com/mnt/export{1...32}
 ```
 
 In this configuration you can see `MINIO_ETCD_ENDPOINTS` points to the etcd backend which manages Minio's
 `config.json` and bucket SRV records. `MINIO_DOMAIN` indicates the domain suffix for the bucket which
 will be used to resolve bucket from DNS. For example if you have a bucket such as `mybucket`, the
-client can use now `mybucket.domain.com` to directly resolve to the right cluster. `MINIO_DOMAIN_IP`
+client can use now `mybucket.domain.com` to directly resolve to the right cluster. `MINIO_PUBLIC_IP`
 points to the public IP address where each cluster might be accessible, this is unique per each cluster.
 
 NOTE: `mybucket` only exists on one cluster either `cluster1` or `cluster2` this is truly random and

pkg/quick/quick.go

@@ -191,10 +191,10 @@ func NewLocalConfig(data interface{}) (Config, error) {
 	return d, nil
 }
 
-// GetVersion - extracts the version information.
+// GetLocalVersion - extracts the version information.
-func GetVersion(filename string) (version string, err error) {
+func GetLocalVersion(filename string) (version string, err error) {
 	var qc Config
-	if qc, err = Load(filename, &struct {
+	if qc, err = LoadLocalConfig(filename, &struct {
 		Version string
 	}{}); err != nil {
 		return "", err
@@ -202,16 +202,16 @@ func GetVersion(filename string) (version string, err error) {
 	return qc.Version(), err
 }
 
-// Load - loads json config from filename for the a given struct data
+// LoadLocalConfig - loads json config from filename for the a given struct data
-func Load(filename string, data interface{}) (qc Config, err error) {
+func LoadLocalConfig(filename string, data interface{}) (qc Config, err error) {
 	if qc, err = NewLocalConfig(data); err == nil {
 		err = qc.Load(filename)
 	}
 	return qc, err
 }
 
-// Save - saves given configuration data into given file as JSON.
+// SaveLocalConfig - saves given configuration data into given file as JSON.
-func Save(filename string, data interface{}) (err error) {
+func SaveLocalConfig(filename string, data interface{}) (err error) {
 	var qc Config
 	if qc, err = NewLocalConfig(data); err == nil {
 		err = qc.Save(filename)

pkg/quick/quick_etcd.go

@@ -71,7 +71,7 @@ func (d etcdConfig) Save(filename string) error {
 
 	kapi := etcdc.NewKeysAPI(d.clnt)
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	_, err = kapi.Create(ctx, filename, string(dataBytes))
+	_, err = kapi.Update(ctx, filename, string(dataBytes))
 	cancel()
 	return err
 }
@@ -173,3 +173,32 @@ func NewEtcdConfig(data interface{}, clnt etcdc.Client) (Config, error) {
 	d.lock = &sync.Mutex{}
 	return d, nil
 }
+
+// GetEtcdVersion - extracts the version information.
+func GetEtcdVersion(filename string, clnt etcdc.Client) (version string, err error) {
+	var qc Config
+	if qc, err = LoadEtcdConfig(filename, &struct {
+		Version string
+	}{}, clnt); err != nil {
+		return "", err
+	}
+	return qc.Version(), err
+}
+
+// LoadEtcdConfig - loads json config from etcd backend for the given struct data
+func LoadEtcdConfig(filename string, data interface{}, clnt etcdc.Client) (qc Config, err error) {
+	if qc, err = NewEtcdConfig(data, clnt); err == nil {
+		err = qc.Load(filename)
+	}
+	return qc, err
+}
+
+// SaveEtcdConfig - saves given configuration data into etcd backend.
+func SaveEtcdConfig(filename string, data interface{}, clnt etcdc.Client) (err error) {
+	var qc Config
+	if qc, err = NewEtcdConfig(data, clnt); err == nil {
+		err = qc.Save(filename)
+	}
+
+	return err
+}

pkg/quick/quick_test.go

@@ -45,7 +45,7 @@ func TestReadVersion(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	version, err := GetVersion("test.json")
+	version, err := GetLocalVersion("test.json")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -69,7 +69,7 @@ func TestReadVersionErr(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = GetVersion("test.json")
+	_, err = GetLocalVersion("test.json")
 	if err == nil {
 		t.Fatal("Unexpected should fail to fetch version")
 	}
@@ -79,7 +79,7 @@ func TestReadVersionErr(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = GetVersion("test.json")
+	_, err = GetLocalVersion("test.json")
 	if err == nil {
 		t.Fatal("Unexpected should fail to fetch version")
 	}
@@ -155,7 +155,7 @@ func TestLoadFile(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{}
-	_, err := Load("test.json", &saveMe)
+	_, err := LoadLocalConfig("test.json", &saveMe)
 	if err == nil {
 		t.Fatal(err)
 	}
@@ -167,7 +167,7 @@ func TestLoadFile(t *testing.T) {
 	if err = file.Close(); err != nil {
 		t.Fatal(err)
 	}
-	_, err = Load("test.json", &saveMe)
+	_, err = LoadLocalConfig("test.json", &saveMe)
 	if err == nil {
 		t.Fatal("Unexpected should fail to load empty JSON")
 	}
@@ -195,7 +195,7 @@ func TestLoadFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	saveMe1 := myStruct{}
-	_, err = Load("test.json", &saveMe1)
+	_, err = LoadLocalConfig("test.json", &saveMe1)
 	if err != nil {
 		t.Fatal(err)
 	}