From 7f214a0e46c24684fd76483f53b53bb31dfee5d3 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 20 Jan 2022 13:03:15 -0800 Subject: [PATCH] use dnscache resolver for resolving command line endpoints (#14135) this helps in caching the resolved values early on, avoids causing further resolution for individual nodes when object layer comes online. this can speed up our startup time during, upgrades etc by an order of magnitude. additional changes in connectLoadInitFormats() and parallelize all calls that might be potentially blocking. --- cmd/bootstrap-peer-server.go | 1 + cmd/common-main.go | 2 +- cmd/endpoint.go | 39 ++++++++++++++++++++++++------------ cmd/net.go | 9 ++++----- cmd/prepare-storage.go | 27 +++++++++++++++++++------ cmd/server-main.go | 37 ++++++++++++++++++++++------------ go.mod | 2 +- go.sum | 4 ++-- 8 files changed, 80 insertions(+), 41 deletions(-) diff --git a/cmd/bootstrap-peer-server.go b/cmd/bootstrap-peer-server.go index 13005d061..8d0e7e701 100644 --- a/cmd/bootstrap-peer-server.go +++ b/cmd/bootstrap-peer-server.go @@ -108,6 +108,7 @@ var skipEnvs = map[string]struct{}{ "MINIO_OPTS": {}, "MINIO_CERT_PASSWD": {}, "MINIO_SERVER_DEBUG": {}, + "MINIO_DSYNC_TRACE": {}, } func getServerSystemCfg() ServerSystemConfig { diff --git a/cmd/common-main.go b/cmd/common-main.go index c3b777a06..11bb3af6c 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -678,7 +678,7 @@ func handleCommonEnvVars() { for _, endpoint := range minioEndpoints { if net.ParseIP(endpoint) == nil { // Checking if the IP is a DNS entry. - addrs, err := net.LookupHost(endpoint) + addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint) if err != nil { logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint) } diff --git a/cmd/endpoint.go b/cmd/endpoint.go index d6088ca06..aadfa17c6 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -643,20 +643,33 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp } } - // Check whether same path is not used in endpoints of a host on different port. - { - pathIPMap := make(map[string]set.StringSet) - for _, endpoint := range endpoints { - host := endpoint.Hostname() - hostIPSet, _ := getHostIP(host) - if IPSet, ok := pathIPMap[endpoint.Path]; ok { - if !IPSet.Intersection(hostIPSet).IsEmpty() { - return endpoints, setupType, - config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path)) + orchestrated := IsKubernetes() || IsDocker() + if !orchestrated { + // Check whether same path is not used in endpoints of a host on different port. + // Only verify this on baremetal setups, DNS is not available in orchestrated + // environments so we can't do much here. + { + pathIPMap := make(map[string]set.StringSet) + hostIPCache := make(map[string]set.StringSet) + for _, endpoint := range endpoints { + host := endpoint.Hostname() + hostIPSet, ok := hostIPCache[host] + if !ok { + hostIPSet, err = getHostIP(host) + if err != nil { + return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err)) + } + hostIPCache[host] = hostIPSet + } + if IPSet, ok := pathIPMap[endpoint.Path]; ok { + if !IPSet.Intersection(hostIPSet).IsEmpty() { + return endpoints, setupType, + config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path)) + } + pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet) + } else { + pathIPMap[endpoint.Path] = hostIPSet } - pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet) - } else { - pathIPMap[endpoint.Path] = hostIPSet } } } diff --git a/cmd/net.go b/cmd/net.go index a3f8c5f6f..f15f68562 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -102,15 +102,14 @@ func mustGetLocalIP6() (ipList set.StringSet) { // getHostIP returns IP address of given host. func getHostIP(host string) (ipList set.StringSet, err error) { - var ips []net.IP - - if ips, err = net.LookupIP(host); err != nil { + addrs, err := globalDNSCache.LookupHost(GlobalContext, host) + if err != nil { return ipList, err } ipList = set.NewStringSet() - for _, ip := range ips { - ipList.Add(ip.String()) + for _, addr := range addrs { + ipList.Add(addr) } return ipList, err diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 1b972c778..0d1ebb5d1 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -179,11 +179,17 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, }(storageDisks) // Sanitize all local disks during server startup. + var wg sync.WaitGroup for _, disk := range storageDisks { if disk != nil && disk.IsLocal() { - disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize() + wg.Add(1) + go func(disk StorageAPI) { + defer wg.Done() + disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize() + }(disk) } } + wg.Wait() for i, err := range errs { if err != nil { @@ -239,13 +245,14 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, // Return error when quorum unformatted disks - indicating we are // waiting for first server to be online. - if quorumUnformattedDisks(sErrs) && !firstDisk { + unformattedDisks := quorumUnformattedDisks(sErrs) + if unformattedDisks && !firstDisk { return nil, nil, errNotFirstDisk } // Return error when quorum unformatted disks but waiting for rest // of the servers to be online. - if quorumUnformattedDisks(sErrs) && firstDisk { + if unformattedDisks && firstDisk { return nil, nil, errFirstDiskWait } @@ -311,10 +318,18 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCou return time.Now().Round(time.Second).Sub(formatStartTime).String() } - // Wait on each try for an update. - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() var tries int + storageDisks, format, err := connectLoadInitFormats(tries, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID, distributionAlgo) + if err == nil { + return storageDisks, format, nil + } + + tries++ // tried already once + + // Wait on each try for an update. + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { select { case <-ticker.C: diff --git a/cmd/server-main.go b/cmd/server-main.go index b1d55ee81..7be9b81d9 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -580,22 +580,23 @@ func serverMain(ctx *cli.Context) { // Initialize users credentials and policies in background right after config has initialized. go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval) - // Initialize transition tier configuration manager - if globalIsErasure { - if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil { - logger.LogIf(GlobalContext, err) - } - } - initDataScanner(GlobalContext, newObject) - if globalIsErasure { // to be done after config init + // Initialize transition tier configuration manager + if globalIsErasure { initBackgroundReplication(GlobalContext, newObject) initBackgroundTransition(GlobalContext, newObject) - globalTierJournal, err = initTierDeletionJournal(GlobalContext) - if err != nil { - logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal") - } + + go func() { + if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil { + logger.LogIf(GlobalContext, err) + } + + globalTierJournal, err = initTierDeletionJournal(GlobalContext) + if err != nil { + logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal") + } + }() } // initialize the new disk cache objects. @@ -609,7 +610,7 @@ func serverMain(ctx *cli.Context) { } // Prints the formatted startup message, if err is not nil then it prints additional information as well. - printStartupMessage(getAPIEndpoints(), err) + go printStartupMessage(getAPIEndpoints(), err) if globalActiveCred.Equal(auth.DefaultCredentials) { msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables", globalActiveCred) @@ -636,7 +637,17 @@ func serverMain(ctx *cli.Context) { if serverDebugLog { logger.Info("== DEBUG Mode enabled ==") logger.Info("Currently set environment settings:") + ks := []string{ + config.EnvAccessKey, + config.EnvSecretKey, + config.EnvRootUser, + config.EnvRootPassword, + } for _, v := range os.Environ() { + // Do not print sensitive creds in debug. + if contains(ks, strings.Split(v, "=")[0]) { + continue + } logger.Info(v) } logger.Info("======") diff --git a/go.mod b/go.mod index 20350f1c7..204da23cc 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/prometheus/client_model v0.2.0 github.com/prometheus/procfs v0.7.3 github.com/rs/cors v1.7.0 - github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 + github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/secure-io/sio-go v0.3.1 github.com/shirou/gopsutil/v3 v3.21.9 github.com/streadway/amqp v1.0.0 diff --git a/go.sum b/go.sum index 78d5468ac..a0c218f61 100644 --- a/go.sum +++ b/go.sum @@ -1356,8 +1356,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 h1:IJ6Df0uxPDtNoByV0KkzVKNseWvZFCNM/S9UoyOMCSI= -github.com/rs/dnscache v0.0.0-20210201191234-295bba877686/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= +github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= +github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=