Allow server to start if one of local nodes in docker/kubernetes setup is resolved (#7452)

Allow server to start if one of the local nodes in docker/kubernetes setup is successfully resolved

- The rule is that we need atleast one local node to work. We dont need to resolve the
  rest at that point.

- In a non-orchestrational setup, we fail if we do not have atleast one local node up
  and running.

- In an orchestrational setup (docker-swarm and kubernetes), We retry with a sleep of 5
  seconds until any one local node shows up.

Fixes #6995

cmd/endpoint-ellipses_test.go

@@ -44,7 +44,6 @@ func TestCreateServerEndpoints(t *testing.T) {
 		{":9000", []string{"/export1{1...32}", "/export1{1...32}"}, false},
 		// Same host cannot export same disk on two ports - special case localhost.
 		{":9001", []string{"http://localhost:900{1...2}/export{1...64}"}, false},
-
 		// Valid inputs.
 		{":9000", []string{"/export1"}, true},
 		{":9000", []string{"/export1", "/export2", "/export3", "/export4"}, true},

cmd/endpoint.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -26,7 +27,9 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
+	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
@@ -44,6 +47,8 @@ const (
 
 	// URLEndpointType - URL style endpoint type enum.
 	URLEndpointType
+
+	retryInterval = 5 // In Seconds.
 )
 
 // Endpoint - any type of endpoint.
@@ -51,6 +56,7 @@ type Endpoint struct {
 	*url.URL
 	IsLocal  bool
 	SetIndex int
+	HostName string
 }
 
 func (endpoint Endpoint) String() string {
@@ -75,6 +81,19 @@ func (endpoint Endpoint) IsHTTPS() bool {
 	return endpoint.Scheme == "https"
 }
 
+// UpdateIsLocal - resolves the host and updates if it is local or not.
+func (endpoint *Endpoint) UpdateIsLocal() error {
+	if !endpoint.IsLocal {
+		isLocal, err := isLocalHost(endpoint.HostName)
+		if err != nil {
+			return err
+		}
+		endpoint.IsLocal = isLocal
+
+	}
+	return nil
+}
+
 // NewEndpoint - returns new endpoint based on given arguments.
 func NewEndpoint(arg string) (ep Endpoint, e error) {
 	// isEmptyPath - check whether given path is not empty.
@@ -87,6 +106,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	}
 
 	var isLocal bool
+	var host string
 	u, err := url.Parse(arg)
 	if err == nil && u.Host != "" {
 		// URL style of endpoint.
@@ -98,7 +118,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			return ep, fmt.Errorf("invalid URL endpoint format")
 		}
 
-		var host, port string
+		var port string
 		host, port, err = net.SplitHostPort(u.Host)
 		if err != nil {
 			if !strings.Contains(err.Error(), "missing port in address") {
@@ -150,10 +170,6 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			}
 		}
 
-		isLocal, err = isLocalHost(host)
-		if err != nil {
-			return ep, err
-		}
 	} else {
 		// Only check if the arg is an ip address and ask for scheme since its absent.
 		// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
@@ -166,8 +182,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	}
 
 	return Endpoint{
-		URL:     u,
-		IsLocal: isLocal,
+		URL:      u,
+		IsLocal:  isLocal,
+		HostName: host,
 	}, nil
 }
 
@@ -200,6 +217,74 @@ func (endpoints EndpointList) GetString(i int) string {
 	return endpoints[i].String()
 }
 
+// UpdateIsLocal - resolves the host and discovers the local host.
+func (endpoints EndpointList) UpdateIsLocal() error {
+	var epsResolved int
+	var foundLocal bool
+	resolvedList := make([]bool, len(endpoints))
+	// Mark the starting time
+	startTime := time.Now()
+	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
+	defer keepAliveTicker.Stop()
+	for {
+		// Break if the local endpoint is found already. Or all the endpoints are resolved.
+		if foundLocal || (epsResolved == len(endpoints)) {
+			break
+		}
+		// Retry infinitely on Kubernetes and Docker swarm.
+		// This is needed as the remote hosts are sometime
+		// not available immediately.
+		select {
+		case <-globalOSSignalCh:
+			return fmt.Errorf("The endpoint resolution got interrupted")
+		default:
+			for i, resolved := range resolvedList {
+				if resolved {
+					continue
+				}
+
+				// return err if not Docker or Kubernetes
+				// We use IsDocker() method to check for Docker Swarm environment
+				// as there is no reliable way to clearly identify Swarm from
+				// Docker environment.
+				isLocal, err := isLocalHost(endpoints[i].HostName)
+				if err != nil {
+					if !IsDocker() && !IsKubernetes() {
+						return err
+					}
+					// time elapsed
+					timeElapsed := time.Since(startTime)
+					// log error only if more than 1s elapsed
+					if timeElapsed > time.Second {
+						// log the message to console about the host not being
+						// resolveable.
+						reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].HostName)
+						reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
+						ctx := logger.SetReqInfo(context.Background(), reqInfo)
+						logger.LogIf(ctx, err)
+					}
+				} else {
+					resolvedList[i] = true
+					endpoints[i].IsLocal = isLocal
+					epsResolved++
+					if !foundLocal {
+						foundLocal = isLocal
+					}
+				}
+			}
+
+			// Wait for the tick, if the there exist a local endpoint in discovery.
+			// Non docker/kubernetes environment does not need to wait.
+			if !foundLocal && (IsDocker() && IsKubernetes()) {
+				<-keepAliveTicker.C
+			}
+		}
+	}
+
+	return nil
+
+}
+
 // localEndpointsMemUsage - returns ServerMemUsageInfo for only the
 // local endpoints from given list of endpoints
 func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
@@ -302,6 +387,7 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 		uniqueArgs.Add(arg)
 		endpoints = append(endpoints, endpoint)
 	}
+
 	return endpoints, nil
 }
 
@@ -341,6 +427,9 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		if err != nil {
 			return serverAddr, endpoints, setupType, err
 		}
+		if err := endpoint.UpdateIsLocal(); err != nil {
+			return serverAddr, endpoints, setupType, err
+		}
 		if endpoint.Type() != PathEndpointType {
 			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
@@ -381,6 +470,10 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, nil
 	}
 
+	if err := endpoints.UpdateIsLocal(); err != nil {
+		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+	}
+
 	// Here all endpoints are URL style.
 	endpointPathSet := set.NewStringSet()
 	localEndpointCount := 0

cmd/endpoint_test.go

@@ -49,11 +49,11 @@ func TestNewEndpoint(t *testing.T) {
 		{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
 		{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
 		{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
-		{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
-		{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
-		{"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil},
-		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
-		{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
+		{"http://localhost/path", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil},
+		{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil},
+		{"https://example.org/path", Endpoint{URL: u2, IsLocal: false, HostName: "example.org"}, URLEndpointType, nil},
+		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true, HostName: "127.0.0.1"}, URLEndpointType, nil},
+		{"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false, HostName: "192.168.253.200"}, URLEndpointType, nil},
 		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
@@ -71,6 +71,10 @@ func TestNewEndpoint(t *testing.T) {
 
 	for _, testCase := range testCases {
 		endpoint, err := NewEndpoint(testCase.arg)
+		if err == nil {
+			err = endpoint.UpdateIsLocal()
+		}
+
 		if testCase.expectedErr == nil {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
@@ -261,46 +265,46 @@ func TestCreateEndpoints(t *testing.T) {
 
 		// DistXL type
 		{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{
-			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
-			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
-			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
-			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
+			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2], HostName: "example.org"},
+			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3], HostName: "example.com"},
 		}, DistXLSetupType, nil},
 
 		{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{
-			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
-			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
-			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
-			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
+			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2], HostName: "example.org"},
+			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3], HostName: "example.com"},
 		}, DistXLSetupType, nil},
 
 		{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", EndpointList{
-			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
-			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
-			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
-			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
+			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1], HostName: "example.org"},
+			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2], HostName: "example.com"},
+			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3], HostName: "example.net"},
 		}, DistXLSetupType, nil},
 
 		{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", EndpointList{
-			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
-			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
-			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
-			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
+			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1], HostName: "example.org"},
+			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2], HostName: "example.com"},
+			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3], HostName: "example.net"},
 		}, DistXLSetupType, nil},
 
 		{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", EndpointList{
-			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
-			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
-			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
-			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
+			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3], HostName: nonLoopBackIP},
 		}, DistXLSetupType, nil},
 
 		// DistXL Setup using only local host.
 		{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", EndpointList{
-			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
-			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
-			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
-			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
+			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0], HostName: "localhost"},
+			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1], HostName: "localhost"},
+			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2], HostName: "127.0.0.1"},
+			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3], HostName: nonLoopBackIP},
 		}, DistXLSetupType, nil},
 	}
 
@@ -357,6 +361,11 @@ func TestGetLocalPeer(t *testing.T) {
 
 	for i, testCase := range testCases {
 		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		if !endpoints[0].IsLocal {
+			if err := endpoints.UpdateIsLocal(); err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		}
 		remotePeer := GetLocalPeer(endpoints)
 		if remotePeer != testCase.expectedResult {
 			t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer)
@@ -384,6 +393,11 @@ func TestGetRemotePeers(t *testing.T) {
 
 	for _, testCase := range testCases {
 		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		if !endpoints[0].IsLocal {
+			if err := endpoints.UpdateIsLocal(); err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		}
 		remotePeers := GetRemotePeers(endpoints)
 		if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
 			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)

cmd/net.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net"
@@ -27,9 +26,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
-	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 )
@@ -102,40 +99,7 @@ func getHostIP(host string) (ipList set.StringSet, err error) {
 	var ips []net.IP
 
 	if ips, err = net.LookupIP(host); err != nil {
-		// return err if not Docker or Kubernetes
-		// We use IsDocker() method to check for Docker Swarm environment
-		// as there is no reliable way to clearly identify Swarm from
-		// Docker environment.
-		if !IsDocker() && !IsKubernetes() {
-			return ipList, err
-		}
-
-		// channel to indicate completion of host resolution
-		doneCh := make(chan struct{})
-		// Indicate retry routine to exit cleanly, upon this function return.
-		defer close(doneCh)
-		// Mark the starting time
-		startTime := time.Now()
-		// wait for hosts to resolve in exponentialbackoff manner
-		for range newRetryTimerSimple(doneCh) {
-			// Retry infinitely on Kubernetes and Docker swarm.
-			// This is needed as the remote hosts are sometime
-			// not available immediately.
-			if ips, err = net.LookupIP(host); err == nil {
-				break
-			}
-			// time elapsed
-			timeElapsed := time.Since(startTime)
-			// log error only if more than 1s elapsed
-			if timeElapsed > time.Second {
-				// log the message to console about the host not being
-				// resolveable.
-				reqInfo := (&logger.ReqInfo{}).AppendTags("host", host)
-				reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
-				logger.LogIf(ctx, err)
-			}
-		}
+		return ipList, err
 	}
 
 	ipList = set.NewStringSet()

cmd/server-main.go

@@ -200,6 +200,8 @@ func serverMain(ctx *cli.Context) {
 		cli.ShowCommandHelpAndExit(ctx, "server", 1)
 	}
 
+	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
+
 	// Disable logging until server initialization is complete, any
 	// error during initialization will be shown as a fatal message
 	logger.Disable = true
@@ -305,8 +307,6 @@ func serverMain(ctx *cli.Context) {
 		globalHTTPServerErrorCh <- globalHTTPServer.Start()
 	}()
 
-	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
-
 	newObject, err := newObjectLayer(globalEndpoints)
 	if err != nil {
 		// Stop watching for any certificate changes.

cmd/storage-rest_test.go

@@ -508,6 +508,10 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES
 		t.Fatalf("NewEndpoint failed %v", endpoint)
 	}
 
+	if err := endpoint.UpdateIsLocal(); err != nil {
+		t.Fatalf("UpdateIsLocal failed %v", err)
+	}
+
 	registerStorageRESTHandlers(router, EndpointList{endpoint})
 	restClient, err := newStorageRESTClient(endpoint)
 	if err != nil {