use dnscache resolver for resolving command line endpoints (#14135)

this helps in caching the resolved values early on, avoids
causing further resolution for individual nodes when
object layer comes online.

this can speed up our startup time during, upgrades etc by
an order of magnitude.

additional changes in connectLoadInitFormats() and parallelize
all calls that might be potentially blocking.

cmd/bootstrap-peer-server.go

@@ -108,6 +108,7 @@ var skipEnvs = map[string]struct{}{
 	"MINIO_OPTS":         {},
 	"MINIO_CERT_PASSWD":  {},
 	"MINIO_SERVER_DEBUG": {},
+	"MINIO_DSYNC_TRACE":  {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {

cmd/common-main.go

@@ -678,7 +678,7 @@ func handleCommonEnvVars() {
 		for _, endpoint := range minioEndpoints {
 			if net.ParseIP(endpoint) == nil {
 				// Checking if the IP is a DNS entry.
-				addrs, err := net.LookupHost(endpoint)
+				addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint)
 				if err != nil {
 					logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
 				}

cmd/endpoint.go

@@ -643,20 +643,33 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		}
 	}
 
-	// Check whether same path is not used in endpoints of a host on different port.
-	{
-		pathIPMap := make(map[string]set.StringSet)
-		for _, endpoint := range endpoints {
-			host := endpoint.Hostname()
-			hostIPSet, _ := getHostIP(host)
-			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
-				if !IPSet.Intersection(hostIPSet).IsEmpty() {
-					return endpoints, setupType,
-						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
+	orchestrated := IsKubernetes() || IsDocker()
+	if !orchestrated {
+		// Check whether same path is not used in endpoints of a host on different port.
+		// Only verify this on baremetal setups, DNS is not available in orchestrated
+		// environments so we can't do much here.
+		{
+			pathIPMap := make(map[string]set.StringSet)
+			hostIPCache := make(map[string]set.StringSet)
+			for _, endpoint := range endpoints {
+				host := endpoint.Hostname()
+				hostIPSet, ok := hostIPCache[host]
+				if !ok {
+					hostIPSet, err = getHostIP(host)
+					if err != nil {
+						return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
+					}
+					hostIPCache[host] = hostIPSet
+				}
+				if IPSet, ok := pathIPMap[endpoint.Path]; ok {
+					if !IPSet.Intersection(hostIPSet).IsEmpty() {
+						return endpoints, setupType,
+							config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
+					}
+					pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
+				} else {
+					pathIPMap[endpoint.Path] = hostIPSet
 				}
-				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
-			} else {
-				pathIPMap[endpoint.Path] = hostIPSet
 			}
 		}
 	}

cmd/net.go

@@ -102,15 +102,14 @@ func mustGetLocalIP6() (ipList set.StringSet) {
 
 // getHostIP returns IP address of given host.
 func getHostIP(host string) (ipList set.StringSet, err error) {
-	var ips []net.IP
-
-	if ips, err = net.LookupIP(host); err != nil {
+	addrs, err := globalDNSCache.LookupHost(GlobalContext, host)
+	if err != nil {
 		return ipList, err
 	}
 
 	ipList = set.NewStringSet()
-	for _, ip := range ips {
-		ipList.Add(ip.String())
+	for _, addr := range addrs {
+		ipList.Add(addr)
 	}
 
 	return ipList, err

cmd/prepare-storage.go

@@ -179,11 +179,17 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 	}(storageDisks)
 
 	// Sanitize all local disks during server startup.
+	var wg sync.WaitGroup
 	for _, disk := range storageDisks {
 		if disk != nil && disk.IsLocal() {
-			disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize()
+			wg.Add(1)
+			go func(disk StorageAPI) {
+				defer wg.Done()
+				disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize()
+			}(disk)
 		}
 	}
+	wg.Wait()
 
 	for i, err := range errs {
 		if err != nil {
@@ -239,13 +245,14 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 
 	// Return error when quorum unformatted disks - indicating we are
 	// waiting for first server to be online.
-	if quorumUnformattedDisks(sErrs) && !firstDisk {
+	unformattedDisks := quorumUnformattedDisks(sErrs)
+	if unformattedDisks && !firstDisk {
 		return nil, nil, errNotFirstDisk
 	}
 
 	// Return error when quorum unformatted disks but waiting for rest
 	// of the servers to be online.
-	if quorumUnformattedDisks(sErrs) && firstDisk {
+	if unformattedDisks && firstDisk {
 		return nil, nil, errFirstDiskWait
 	}
 
@@ -311,10 +318,18 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCou
 		return time.Now().Round(time.Second).Sub(formatStartTime).String()
 	}
 
-	// Wait on each try for an update.
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
 	var tries int
+	storageDisks, format, err := connectLoadInitFormats(tries, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID, distributionAlgo)
+	if err == nil {
+		return storageDisks, format, nil
+	}
+
+	tries++ // tried already once
+
+	// Wait on each try for an update.
+	ticker := time.NewTicker(250 * time.Millisecond)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-ticker.C:

cmd/server-main.go

@@ -580,22 +580,23 @@ func serverMain(ctx *cli.Context) {
 	// Initialize users credentials and policies in background right after config has initialized.
 	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)
 
-	// Initialize transition tier configuration manager
-	if globalIsErasure {
-		if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
-			logger.LogIf(GlobalContext, err)
-		}
-	}
-
 	initDataScanner(GlobalContext, newObject)
 
-	if globalIsErasure { // to be done after config init
+	// Initialize transition tier configuration manager
+	if globalIsErasure {
 		initBackgroundReplication(GlobalContext, newObject)
 		initBackgroundTransition(GlobalContext, newObject)
-		globalTierJournal, err = initTierDeletionJournal(GlobalContext)
-		if err != nil {
-			logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
-		}
+
+		go func() {
+			if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
+				logger.LogIf(GlobalContext, err)
+			}
+
+			globalTierJournal, err = initTierDeletionJournal(GlobalContext)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
+			}
+		}()
 	}
 
 	// initialize the new disk cache objects.
@@ -609,7 +610,7 @@ func serverMain(ctx *cli.Context) {
 	}
 
 	// Prints the formatted startup message, if err is not nil then it prints additional information as well.
-	printStartupMessage(getAPIEndpoints(), err)
+	go printStartupMessage(getAPIEndpoints(), err)
 
 	if globalActiveCred.Equal(auth.DefaultCredentials) {
 		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables", globalActiveCred)
@@ -636,7 +637,17 @@ func serverMain(ctx *cli.Context) {
 	if serverDebugLog {
 		logger.Info("== DEBUG Mode enabled ==")
 		logger.Info("Currently set environment settings:")
+		ks := []string{
+			config.EnvAccessKey,
+			config.EnvSecretKey,
+			config.EnvRootUser,
+			config.EnvRootPassword,
+		}
 		for _, v := range os.Environ() {
+			// Do not print sensitive creds in debug.
+			if contains(ks, strings.Split(v, "=")[0]) {
+				continue
+			}
 			logger.Info(v)
 		}
 		logger.Info("======")

go.mod

@@ -72,7 +72,7 @@ require (
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/procfs v0.7.3
 	github.com/rs/cors v1.7.0
-	github.com/rs/dnscache v0.0.0-20210201191234-295bba877686
+	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/secure-io/sio-go v0.3.1
 	github.com/shirou/gopsutil/v3 v3.21.9
 	github.com/streadway/amqp v1.0.0

go.sum

@@ -1356,8 +1356,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
-github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 h1:IJ6Df0uxPDtNoByV0KkzVKNseWvZFCNM/S9UoyOMCSI=
-github.com/rs/dnscache v0.0.0-20210201191234-295bba877686/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
+github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs=
+github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=