use dnscache resolver for resolving command line endpoints (#14135)

this helps in caching the resolved values early on, avoids
causing further resolution for individual nodes when
object layer comes online.

this can speed up our startup time during, upgrades etc by
an order of magnitude.

additional changes in connectLoadInitFormats() and parallelize
all calls that might be potentially blocking.
This commit is contained in:
Harshavardhana 2022-01-20 13:03:15 -08:00 committed by GitHub
parent e1a0a1e73c
commit 7f214a0e46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 80 additions and 41 deletions

View File

@ -108,6 +108,7 @@ var skipEnvs = map[string]struct{}{
"MINIO_OPTS": {}, "MINIO_OPTS": {},
"MINIO_CERT_PASSWD": {}, "MINIO_CERT_PASSWD": {},
"MINIO_SERVER_DEBUG": {}, "MINIO_SERVER_DEBUG": {},
"MINIO_DSYNC_TRACE": {},
} }
func getServerSystemCfg() ServerSystemConfig { func getServerSystemCfg() ServerSystemConfig {

View File

@ -678,7 +678,7 @@ func handleCommonEnvVars() {
for _, endpoint := range minioEndpoints { for _, endpoint := range minioEndpoints {
if net.ParseIP(endpoint) == nil { if net.ParseIP(endpoint) == nil {
// Checking if the IP is a DNS entry. // Checking if the IP is a DNS entry.
addrs, err := net.LookupHost(endpoint) addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint)
if err != nil { if err != nil {
logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint) logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
} }

View File

@ -643,20 +643,33 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
} }
} }
// Check whether same path is not used in endpoints of a host on different port. orchestrated := IsKubernetes() || IsDocker()
{ if !orchestrated {
pathIPMap := make(map[string]set.StringSet) // Check whether same path is not used in endpoints of a host on different port.
for _, endpoint := range endpoints { // Only verify this on baremetal setups, DNS is not available in orchestrated
host := endpoint.Hostname() // environments so we can't do much here.
hostIPSet, _ := getHostIP(host) {
if IPSet, ok := pathIPMap[endpoint.Path]; ok { pathIPMap := make(map[string]set.StringSet)
if !IPSet.Intersection(hostIPSet).IsEmpty() { hostIPCache := make(map[string]set.StringSet)
return endpoints, setupType, for _, endpoint := range endpoints {
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path)) host := endpoint.Hostname()
hostIPSet, ok := hostIPCache[host]
if !ok {
hostIPSet, err = getHostIP(host)
if err != nil {
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
}
hostIPCache[host] = hostIPSet
}
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
if !IPSet.Intersection(hostIPSet).IsEmpty() {
return endpoints, setupType,
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
}
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
} else {
pathIPMap[endpoint.Path] = hostIPSet
} }
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
} else {
pathIPMap[endpoint.Path] = hostIPSet
} }
} }
} }

View File

@ -102,15 +102,14 @@ func mustGetLocalIP6() (ipList set.StringSet) {
// getHostIP returns IP address of given host. // getHostIP returns IP address of given host.
func getHostIP(host string) (ipList set.StringSet, err error) { func getHostIP(host string) (ipList set.StringSet, err error) {
var ips []net.IP addrs, err := globalDNSCache.LookupHost(GlobalContext, host)
if err != nil {
if ips, err = net.LookupIP(host); err != nil {
return ipList, err return ipList, err
} }
ipList = set.NewStringSet() ipList = set.NewStringSet()
for _, ip := range ips { for _, addr := range addrs {
ipList.Add(ip.String()) ipList.Add(addr)
} }
return ipList, err return ipList, err

View File

@ -179,11 +179,17 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
}(storageDisks) }(storageDisks)
// Sanitize all local disks during server startup. // Sanitize all local disks during server startup.
var wg sync.WaitGroup
for _, disk := range storageDisks { for _, disk := range storageDisks {
if disk != nil && disk.IsLocal() { if disk != nil && disk.IsLocal() {
disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize() wg.Add(1)
go func(disk StorageAPI) {
defer wg.Done()
disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize()
}(disk)
} }
} }
wg.Wait()
for i, err := range errs { for i, err := range errs {
if err != nil { if err != nil {
@ -239,13 +245,14 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
// Return error when quorum unformatted disks - indicating we are // Return error when quorum unformatted disks - indicating we are
// waiting for first server to be online. // waiting for first server to be online.
if quorumUnformattedDisks(sErrs) && !firstDisk { unformattedDisks := quorumUnformattedDisks(sErrs)
if unformattedDisks && !firstDisk {
return nil, nil, errNotFirstDisk return nil, nil, errNotFirstDisk
} }
// Return error when quorum unformatted disks but waiting for rest // Return error when quorum unformatted disks but waiting for rest
// of the servers to be online. // of the servers to be online.
if quorumUnformattedDisks(sErrs) && firstDisk { if unformattedDisks && firstDisk {
return nil, nil, errFirstDiskWait return nil, nil, errFirstDiskWait
} }
@ -311,10 +318,18 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCou
return time.Now().Round(time.Second).Sub(formatStartTime).String() return time.Now().Round(time.Second).Sub(formatStartTime).String()
} }
// Wait on each try for an update.
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
var tries int var tries int
storageDisks, format, err := connectLoadInitFormats(tries, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID, distributionAlgo)
if err == nil {
return storageDisks, format, nil
}
tries++ // tried already once
// Wait on each try for an update.
ticker := time.NewTicker(250 * time.Millisecond)
defer ticker.Stop()
for { for {
select { select {
case <-ticker.C: case <-ticker.C:

View File

@ -580,22 +580,23 @@ func serverMain(ctx *cli.Context) {
// Initialize users credentials and policies in background right after config has initialized. // Initialize users credentials and policies in background right after config has initialized.
go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval) go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)
// Initialize transition tier configuration manager
if globalIsErasure {
if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
logger.LogIf(GlobalContext, err)
}
}
initDataScanner(GlobalContext, newObject) initDataScanner(GlobalContext, newObject)
if globalIsErasure { // to be done after config init // Initialize transition tier configuration manager
if globalIsErasure {
initBackgroundReplication(GlobalContext, newObject) initBackgroundReplication(GlobalContext, newObject)
initBackgroundTransition(GlobalContext, newObject) initBackgroundTransition(GlobalContext, newObject)
globalTierJournal, err = initTierDeletionJournal(GlobalContext)
if err != nil { go func() {
logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal") if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
} logger.LogIf(GlobalContext, err)
}
globalTierJournal, err = initTierDeletionJournal(GlobalContext)
if err != nil {
logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
}
}()
} }
// initialize the new disk cache objects. // initialize the new disk cache objects.
@ -609,7 +610,7 @@ func serverMain(ctx *cli.Context) {
} }
// Prints the formatted startup message, if err is not nil then it prints additional information as well. // Prints the formatted startup message, if err is not nil then it prints additional information as well.
printStartupMessage(getAPIEndpoints(), err) go printStartupMessage(getAPIEndpoints(), err)
if globalActiveCred.Equal(auth.DefaultCredentials) { if globalActiveCred.Equal(auth.DefaultCredentials) {
msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables", globalActiveCred) msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables", globalActiveCred)
@ -636,7 +637,17 @@ func serverMain(ctx *cli.Context) {
if serverDebugLog { if serverDebugLog {
logger.Info("== DEBUG Mode enabled ==") logger.Info("== DEBUG Mode enabled ==")
logger.Info("Currently set environment settings:") logger.Info("Currently set environment settings:")
ks := []string{
config.EnvAccessKey,
config.EnvSecretKey,
config.EnvRootUser,
config.EnvRootPassword,
}
for _, v := range os.Environ() { for _, v := range os.Environ() {
// Do not print sensitive creds in debug.
if contains(ks, strings.Split(v, "=")[0]) {
continue
}
logger.Info(v) logger.Info(v)
} }
logger.Info("======") logger.Info("======")

2
go.mod
View File

@ -72,7 +72,7 @@ require (
github.com/prometheus/client_model v0.2.0 github.com/prometheus/client_model v0.2.0
github.com/prometheus/procfs v0.7.3 github.com/prometheus/procfs v0.7.3
github.com/rs/cors v1.7.0 github.com/rs/cors v1.7.0
github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
github.com/secure-io/sio-go v0.3.1 github.com/secure-io/sio-go v0.3.1
github.com/shirou/gopsutil/v3 v3.21.9 github.com/shirou/gopsutil/v3 v3.21.9
github.com/streadway/amqp v1.0.0 github.com/streadway/amqp v1.0.0

4
go.sum
View File

@ -1356,8 +1356,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 h1:IJ6Df0uxPDtNoByV0KkzVKNseWvZFCNM/S9UoyOMCSI= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs=
github.com/rs/dnscache v0.0.0-20210201191234-295bba877686/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=