mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
use dnscache resolver for resolving command line endpoints (#14135)
this helps in caching the resolved values early on, avoids causing further resolution for individual nodes when object layer comes online. this can speed up our startup time during, upgrades etc by an order of magnitude. additional changes in connectLoadInitFormats() and parallelize all calls that might be potentially blocking.
This commit is contained in:
parent
e1a0a1e73c
commit
7f214a0e46
@ -108,6 +108,7 @@ var skipEnvs = map[string]struct{}{
|
||||
"MINIO_OPTS": {},
|
||||
"MINIO_CERT_PASSWD": {},
|
||||
"MINIO_SERVER_DEBUG": {},
|
||||
"MINIO_DSYNC_TRACE": {},
|
||||
}
|
||||
|
||||
func getServerSystemCfg() ServerSystemConfig {
|
||||
|
@ -678,7 +678,7 @@ func handleCommonEnvVars() {
|
||||
for _, endpoint := range minioEndpoints {
|
||||
if net.ParseIP(endpoint) == nil {
|
||||
// Checking if the IP is a DNS entry.
|
||||
addrs, err := net.LookupHost(endpoint)
|
||||
addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint)
|
||||
if err != nil {
|
||||
logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
|
||||
}
|
||||
|
@ -643,16 +643,28 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
|
||||
}
|
||||
}
|
||||
|
||||
orchestrated := IsKubernetes() || IsDocker()
|
||||
if !orchestrated {
|
||||
// Check whether same path is not used in endpoints of a host on different port.
|
||||
// Only verify this on baremetal setups, DNS is not available in orchestrated
|
||||
// environments so we can't do much here.
|
||||
{
|
||||
pathIPMap := make(map[string]set.StringSet)
|
||||
hostIPCache := make(map[string]set.StringSet)
|
||||
for _, endpoint := range endpoints {
|
||||
host := endpoint.Hostname()
|
||||
hostIPSet, _ := getHostIP(host)
|
||||
hostIPSet, ok := hostIPCache[host]
|
||||
if !ok {
|
||||
hostIPSet, err = getHostIP(host)
|
||||
if err != nil {
|
||||
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
|
||||
}
|
||||
hostIPCache[host] = hostIPSet
|
||||
}
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
return endpoints, setupType,
|
||||
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
|
||||
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
|
||||
}
|
||||
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
|
||||
} else {
|
||||
@ -660,6 +672,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check whether same path is used for more than 1 local endpoints.
|
||||
{
|
||||
|
@ -102,15 +102,14 @@ func mustGetLocalIP6() (ipList set.StringSet) {
|
||||
|
||||
// getHostIP returns IP address of given host.
|
||||
func getHostIP(host string) (ipList set.StringSet, err error) {
|
||||
var ips []net.IP
|
||||
|
||||
if ips, err = net.LookupIP(host); err != nil {
|
||||
addrs, err := globalDNSCache.LookupHost(GlobalContext, host)
|
||||
if err != nil {
|
||||
return ipList, err
|
||||
}
|
||||
|
||||
ipList = set.NewStringSet()
|
||||
for _, ip := range ips {
|
||||
ipList.Add(ip.String())
|
||||
for _, addr := range addrs {
|
||||
ipList.Add(addr)
|
||||
}
|
||||
|
||||
return ipList, err
|
||||
|
@ -179,11 +179,17 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
|
||||
}(storageDisks)
|
||||
|
||||
// Sanitize all local disks during server startup.
|
||||
var wg sync.WaitGroup
|
||||
for _, disk := range storageDisks {
|
||||
if disk != nil && disk.IsLocal() {
|
||||
wg.Add(1)
|
||||
go func(disk StorageAPI) {
|
||||
defer wg.Done()
|
||||
disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize()
|
||||
}(disk)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
@ -239,13 +245,14 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
|
||||
|
||||
// Return error when quorum unformatted disks - indicating we are
|
||||
// waiting for first server to be online.
|
||||
if quorumUnformattedDisks(sErrs) && !firstDisk {
|
||||
unformattedDisks := quorumUnformattedDisks(sErrs)
|
||||
if unformattedDisks && !firstDisk {
|
||||
return nil, nil, errNotFirstDisk
|
||||
}
|
||||
|
||||
// Return error when quorum unformatted disks but waiting for rest
|
||||
// of the servers to be online.
|
||||
if quorumUnformattedDisks(sErrs) && firstDisk {
|
||||
if unformattedDisks && firstDisk {
|
||||
return nil, nil, errFirstDiskWait
|
||||
}
|
||||
|
||||
@ -311,10 +318,18 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCou
|
||||
return time.Now().Round(time.Second).Sub(formatStartTime).String()
|
||||
}
|
||||
|
||||
// Wait on each try for an update.
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
var tries int
|
||||
storageDisks, format, err := connectLoadInitFormats(tries, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID, distributionAlgo)
|
||||
if err == nil {
|
||||
return storageDisks, format, nil
|
||||
}
|
||||
|
||||
tries++ // tried already once
|
||||
|
||||
// Wait on each try for an update.
|
||||
ticker := time.NewTicker(250 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
|
@ -580,22 +580,23 @@ func serverMain(ctx *cli.Context) {
|
||||
// Initialize users credentials and policies in background right after config has initialized.
|
||||
go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)
|
||||
|
||||
initDataScanner(GlobalContext, newObject)
|
||||
|
||||
// Initialize transition tier configuration manager
|
||||
if globalIsErasure {
|
||||
initBackgroundReplication(GlobalContext, newObject)
|
||||
initBackgroundTransition(GlobalContext, newObject)
|
||||
|
||||
go func() {
|
||||
if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
}
|
||||
}
|
||||
|
||||
initDataScanner(GlobalContext, newObject)
|
||||
|
||||
if globalIsErasure { // to be done after config init
|
||||
initBackgroundReplication(GlobalContext, newObject)
|
||||
initBackgroundTransition(GlobalContext, newObject)
|
||||
globalTierJournal, err = initTierDeletionJournal(GlobalContext)
|
||||
if err != nil {
|
||||
logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// initialize the new disk cache objects.
|
||||
@ -609,7 +610,7 @@ func serverMain(ctx *cli.Context) {
|
||||
}
|
||||
|
||||
// Prints the formatted startup message, if err is not nil then it prints additional information as well.
|
||||
printStartupMessage(getAPIEndpoints(), err)
|
||||
go printStartupMessage(getAPIEndpoints(), err)
|
||||
|
||||
if globalActiveCred.Equal(auth.DefaultCredentials) {
|
||||
msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables", globalActiveCred)
|
||||
@ -636,7 +637,17 @@ func serverMain(ctx *cli.Context) {
|
||||
if serverDebugLog {
|
||||
logger.Info("== DEBUG Mode enabled ==")
|
||||
logger.Info("Currently set environment settings:")
|
||||
ks := []string{
|
||||
config.EnvAccessKey,
|
||||
config.EnvSecretKey,
|
||||
config.EnvRootUser,
|
||||
config.EnvRootPassword,
|
||||
}
|
||||
for _, v := range os.Environ() {
|
||||
// Do not print sensitive creds in debug.
|
||||
if contains(ks, strings.Split(v, "=")[0]) {
|
||||
continue
|
||||
}
|
||||
logger.Info(v)
|
||||
}
|
||||
logger.Info("======")
|
||||
|
2
go.mod
2
go.mod
@ -72,7 +72,7 @@ require (
|
||||
github.com/prometheus/client_model v0.2.0
|
||||
github.com/prometheus/procfs v0.7.3
|
||||
github.com/rs/cors v1.7.0
|
||||
github.com/rs/dnscache v0.0.0-20210201191234-295bba877686
|
||||
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
|
||||
github.com/secure-io/sio-go v0.3.1
|
||||
github.com/shirou/gopsutil/v3 v3.21.9
|
||||
github.com/streadway/amqp v1.0.0
|
||||
|
4
go.sum
4
go.sum
@ -1356,8 +1356,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
|
||||
github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
|
||||
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 h1:IJ6Df0uxPDtNoByV0KkzVKNseWvZFCNM/S9UoyOMCSI=
|
||||
github.com/rs/dnscache v0.0.0-20210201191234-295bba877686/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
|
||||
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs=
|
||||
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
|
||||
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
|
Loading…
Reference in New Issue
Block a user