allow configuring maximum idle connections per host (#18908)

cmd/common-main.go

@@ -397,6 +397,7 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
 	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
 	ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
+	ctxt.MaxIdleConnsPerHost = ctx.Int("max-idle-conns-per-host")
 
 	if conf := ctx.String("config"); len(conf) > 0 {
 		err = mergeServerCtxtFromConfigFile(conf, ctxt)

cmd/globals.go

@@ -157,9 +157,10 @@ type serverCtxt struct {
 	ConnReadDeadline  time.Duration
 	ConnWriteDeadline time.Duration
 
-	ShutdownTimeout   time.Duration
+	ShutdownTimeout     time.Duration
-	IdleTimeout       time.Duration
+	IdleTimeout         time.Duration
-	ReadHeaderTimeout time.Duration
+	ReadHeaderTimeout   time.Duration
+	MaxIdleConnsPerHost int
 
 	// The layout of disks as interpreted
 	Layout disksLayout

cmd/server-main.go

@@ -135,6 +135,13 @@ var ServerFlags = []cli.Flag{
 		Value:  10 * time.Minute,
 		EnvVar: "MINIO_DNS_CACHE_TTL",
 	},
+	cli.IntFlag{
+		Name:   "max-idle-conns-per-host",
+		Usage:  "set a custom max idle connections per host value",
+		Hidden: true,
+		Value:  2048,
+		EnvVar: "MINIO_MAX_IDLE_CONNS_PER_HOST",
+	},
 	cli.StringSliceFlag{
 		Name:  "ftp",
 		Usage: "enable and configure an FTP(Secure) server",
@@ -330,7 +337,7 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 	// allow transport to be HTTP/1.1 for proxying.
 	globalProxyTransport = NewCustomHTTPProxyTransport()()
 	globalProxyEndpoints = GetProxyEndpoints(globalEndpoints)
-	globalInternodeTransport = NewInternodeHTTPTransport()()
+	globalInternodeTransport = NewInternodeHTTPTransport(ctxt.MaxIdleConnsPerHost)()
 	globalRemoteTargetTransport = NewRemoteTargetHTTPTransport(false)()
 
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{

cmd/test-utils_test.go

@@ -111,7 +111,7 @@ func TestMain(m *testing.M) {
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background())
 
-	globalInternodeTransport = NewInternodeHTTPTransport()()
+	globalInternodeTransport = NewInternodeHTTPTransport(0)()
 
 	initHelp()
 

cmd/utils.go

@@ -587,7 +587,7 @@ func GetDefaultConnSettings() xhttp.ConnSettings {
 
 // NewInternodeHTTPTransport returns a transport for internode MinIO
 // connections.
-func NewInternodeHTTPTransport() func() http.RoundTripper {
+func NewInternodeHTTPTransport(maxIdleConnsPerHost int) func() http.RoundTripper {
 	lookupHost := globalDNSCache.LookupHost
 	if IsKubernetes() || IsDocker() {
 		lookupHost = nil
@@ -601,7 +601,7 @@ func NewInternodeHTTPTransport() func() http.RoundTripper {
 		CurvePreferences: fips.TLSCurveIDs(),
 		EnableHTTP2:      false,
 		TCPOptions:       globalTCPOptions,
-	}.NewInternodeHTTPTransport()
+	}.NewInternodeHTTPTransport(maxIdleConnsPerHost)
 }
 
 // NewCustomHTTPProxyTransport is used only for proxied requests, specifically

internal/http/transports.go

@@ -49,7 +49,11 @@ type ConnSettings struct {
 	TCPOptions TCPOptions
 }
 
-func (s ConnSettings) getDefaultTransport() *http.Transport {
+func (s ConnSettings) getDefaultTransport(maxIdleConnsPerHost int) *http.Transport {
+	if maxIdleConnsPerHost <= 0 {
+		maxIdleConnsPerHost = 1024
+	}
+
 	dialContext := s.DialContext
 	if dialContext == nil {
 		dialContext = DialContextWithLookupHost(s.LookupHost, NewInternodeDialContext(s.DialTimeout, s.TCPOptions))
@@ -67,7 +71,7 @@ func (s ConnSettings) getDefaultTransport() *http.Transport {
 	tr := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		DialContext:           dialContext,
-		MaxIdleConnsPerHost:   1024,
+		MaxIdleConnsPerHost:   maxIdleConnsPerHost,
 		WriteBufferSize:       32 << 10, // 32KiB moving up from 4KiB default
 		ReadBufferSize:        32 << 10, // 32KiB moving up from 4KiB default
 		IdleConnTimeout:       15 * time.Second,
@@ -106,8 +110,8 @@ func (s ConnSettings) getDefaultTransport() *http.Transport {
 }
 
 // NewInternodeHTTPTransport returns transport for internode MinIO connections.
-func (s ConnSettings) NewInternodeHTTPTransport() func() http.RoundTripper {
+func (s ConnSettings) NewInternodeHTTPTransport(maxIdleConnsPerHost int) func() http.RoundTripper {
-	tr := s.getDefaultTransport()
+	tr := s.getDefaultTransport(maxIdleConnsPerHost)
 
 	// Settings specific to internode requests.
 	tr.TLSHandshakeTimeout = 15 * time.Second
@@ -121,7 +125,7 @@ func (s ConnSettings) NewInternodeHTTPTransport() func() http.RoundTripper {
 // only supports HTTP/1.1
 func (s ConnSettings) NewCustomHTTPProxyTransport() func() *http.Transport {
 	s.EnableHTTP2 = false
-	tr := s.getDefaultTransport()
+	tr := s.getDefaultTransport(0)
 
 	// Settings specific to proxied requests.
 	tr.ResponseHeaderTimeout = 30 * time.Minute
@@ -133,7 +137,7 @@ func (s ConnSettings) NewCustomHTTPProxyTransport() func() *http.Transport {
 
 // NewHTTPTransportWithTimeout allows setting a timeout for response headers
 func (s ConnSettings) NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport {
-	tr := s.getDefaultTransport()
+	tr := s.getDefaultTransport(0)
 
 	// Settings specific to this transport.
 	tr.ResponseHeaderTimeout = timeout
@@ -161,7 +165,7 @@ func (s ConnSettings) NewHTTPTransportWithClientCerts(ctx context.Context, clien
 // NewRemoteTargetHTTPTransport returns a new http configuration
 // used while communicating with the remote replication targets.
 func (s ConnSettings) NewRemoteTargetHTTPTransport(insecure bool) func() *http.Transport {
-	tr := s.getDefaultTransport()
+	tr := s.getDefaultTransport(0)
 
 	tr.TLSHandshakeTimeout = 10 * time.Second
 	tr.ResponseHeaderTimeout = 0