add configurable VRF interface and user-timeout (#17108)

cmd/globals.go

@@ -212,6 +212,7 @@ var (
 	globalTLSCerts *certs.Manager
 
 	globalHTTPServer        *xhttp.Server
+	globalTCPOptions        xhttp.TCPOptions
 	globalHTTPServerErrorCh = make(chan error)
 	globalOSSignalCh        = make(chan os.Signal, 1)
 

cmd/net.go

@@ -207,19 +207,6 @@ func isHostIP(ipAddress string) bool {
 	return net.ParseIP(host) != nil
 }
 
-// checkPortAvailability - check if given host and port is already in use.
-// Note: The check method tries to listen on given port and closes it.
-// It is possible to have a disconnected client in this tiny window of time.
-func checkPortAvailability(host, port string) (err error) {
-	l, err := net.Listen("tcp", net.JoinHostPort(host, port))
-	if err != nil {
-		return err
-	}
-	// As we are able to listen on this network, the port is not in use.
-	// Close the listener and continue check other networks.
-	return l.Close()
-}
-
 // extractHostPort - extracts host/port from many address formats
 // such as, ":9000", "localhost:9000", "http://localhost:9000/"
 func extractHostPort(hostAddr string) (string, string, error) {

cmd/net_test.go

@@ -20,9 +20,7 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"net"
 	"reflect"
-	"runtime"
 	"testing"
 
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -180,61 +178,6 @@ func TestGetAPIEndpoints(t *testing.T) {
 	}
 }
 
-// Ask the kernel for a free open port.
-func getFreePort() string {
-	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
-	if err != nil {
-		panic(err)
-	}
-
-	l, err := net.ListenTCP("tcp", addr)
-	if err != nil {
-		panic(err)
-	}
-	defer l.Close()
-	return fmt.Sprintf("%d", l.Addr().(*net.TCPAddr).Port)
-}
-
-// Tests for port availability logic written for server startup sequence.
-func TestCheckPortAvailability(t *testing.T) {
-	// Make a port is not available.
-	port := getFreePort()
-	listener, err := net.Listen("tcp", net.JoinHostPort("", port))
-	if err != nil {
-		t.Fatalf("Unable to listen on port %v", port)
-	}
-	defer listener.Close()
-
-	testCases := []struct {
-		host        string
-		port        string
-		expectedErr error
-	}{
-		{"", port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
-		{"127.0.0.1", port, fmt.Errorf("listen tcp 127.0.0.1:%v: bind: address already in use", port)},
-		{"", getFreePort(), nil},
-	}
-
-	for _, testCase := range testCases {
-		// On MS Windows and Mac, skip checking error case due to https://github.com/golang/go/issues/7598
-		if (runtime.GOOS == globalWindowsOSName || runtime.GOOS == globalMacOSName || runtime.GOOS == "solaris") && testCase.expectedErr != nil {
-			continue
-		}
-
-		err := checkPortAvailability(testCase.host, testCase.port)
-		switch {
-		case testCase.expectedErr == nil:
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		case err == nil:
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		case testCase.expectedErr.Error() != err.Error():
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-		}
-	}
-}
-
 func TestCheckLocalServerAddr(t *testing.T) {
 	testCases := []struct {
 		serverAddr  string

cmd/server-main.go

@@ -107,6 +107,19 @@ var ServerFlags = []cli.Flag{
 		Value:  10 * time.Minute,
 		EnvVar: "MINIO_CONN_WRITE_DEADLINE",
 	},
+	cli.DurationFlag{
+		Name:   "conn-user-timeout",
+		Usage:  "custom TCP_USER_TIMEOUT for socket buffers",
+		Hidden: true,
+		Value:  10 * time.Minute,
+		EnvVar: "MINIO_CONN_USER_TIMEOUT",
+	},
+	cli.StringFlag{
+		Name:   "interface",
+		Usage:  "bind to right VRF device for MinIO services",
+		Hidden: true,
+		EnvVar: "MINIO_INTERFACE",
+	},
 	cli.StringSliceFlag{
 		Name:  "ftp",
 		Usage: "enable and configure an FTP(Secure) server",
@@ -264,11 +277,16 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 		},
 	})
 
+	globalTCPOptions = xhttp.TCPOptions{
+		UserTimeout: int(ctx.Duration("conn-user-timeout").Milliseconds()),
+		Interface:   ctx.String("interface"),
+	}
+
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
 	// (non-)minio process is listening on IPv4 of given port.
 	// To avoid this error situation we check for port availability.
-	logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the server")
+	logger.FatalIf(xhttp.CheckPortAvailability(globalMinioHost, globalMinioPort, globalTCPOptions), "Unable to start the server")
 
 	globalIsErasure = (setupType == ErasureSetupType)
 	globalIsDistErasure = (setupType == DistErasureSetupType)
@@ -570,7 +588,8 @@ func serverMain(ctx *cli.Context) {
 		UseIdleTimeout(ctx.Duration("idle-timeout")).
 		UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
 		UseBaseContext(GlobalContext).
-		UseCustomLogger(log.New(io.Discard, "", 0)) // Turn-off random logging by Go stdlib
+		UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
+		UseTCPOptions(globalTCPOptions)
 
 	go func() {
 		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)

cmd/update.go

@@ -403,7 +403,7 @@ func parseReleaseData(data string) (sha256Sum []byte, releaseTime time.Time, rel
 func getUpdateTransport(timeout time.Duration) http.RoundTripper {
 	var updateTransport http.RoundTripper = &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           xhttp.NewCustomDialContext(timeout),
+		DialContext:           xhttp.NewCustomDialContext(timeout, globalTCPOptions),
 		IdleConnTimeout:       timeout,
 		TLSHandshakeTimeout:   timeout,
 		ExpectContinueTimeout: timeout,

cmd/utils.go

@@ -574,6 +574,7 @@ func GetDefaultConnSettings() xhttp.ConnSettings {
 		DNSCache:    globalDNSCache,
 		DialTimeout: rest.DefaultTimeout,
 		RootCAs:     globalRootCAs,
+		TCPOptions:  globalTCPOptions,
 	}
 }
 
@@ -587,6 +588,7 @@ func NewInternodeHTTPTransport() func() http.RoundTripper {
 		CipherSuites:     fips.TLSCiphers(),
 		CurvePreferences: fips.TLSCurveIDs(),
 		EnableHTTP2:      false,
+		TCPOptions:       globalTCPOptions,
 	}.NewInternodeHTTPTransport()
 }
 
@@ -600,6 +602,7 @@ func NewCustomHTTPProxyTransport() func() *http.Transport {
 		CipherSuites:     fips.TLSCiphers(),
 		CurvePreferences: fips.TLSCurveIDs(),
 		EnableHTTP2:      false,
+		TCPOptions:       globalTCPOptions,
 	}.NewCustomHTTPProxyTransport()
 }
 
@@ -610,6 +613,7 @@ func NewHTTPTransportWithClientCerts(clientCert, clientKey string) *http.Transpo
 		DNSCache:    globalDNSCache,
 		DialTimeout: defaultDialTimeout,
 		RootCAs:     globalRootCAs,
+		TCPOptions:  globalTCPOptions,
 		EnableHTTP2: false,
 	}
 
@@ -643,6 +647,7 @@ func NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport {
 		DNSCache:    globalDNSCache,
 		DialTimeout: defaultDialTimeout,
 		RootCAs:     globalRootCAs,
+		TCPOptions:  globalTCPOptions,
 		EnableHTTP2: false,
 	}.NewHTTPTransportWithTimeout(timeout)
 }
@@ -677,6 +682,7 @@ func NewRemoteTargetHTTPTransport() func() *http.Transport {
 		DialContext: newCustomDialContext(),
 		DNSCache:    globalDNSCache,
 		RootCAs:     globalRootCAs,
+		TCPOptions:  globalTCPOptions,
 		EnableHTTP2: false,
 	}.NewRemoteTargetHTTPTransport()
 }