Add response header timeouts (#9170)

- Add conservative timeouts upto 3 minutes
  for internode communication
- Add aggressive timeouts of 30 seconds
  for gateway communication

Fixes #9105
Fixes #8732
Fixes #8881
Fixes #8376
Fixes #9028

cmd/utils.go

@@ -151,10 +151,9 @@ const (
 	// (Acceptable values range from 1 to 10000 inclusive)
 	globalMaxPartID = 10000
 
-	// Default values used while communicating for
-	// internode communication.
-	defaultDialTimeout   = 15 * time.Second
-	defaultDialKeepAlive = 20 * time.Second
+	// Default values used while communicating for internode communication.
+	defaultDialTimeout   = 5 * time.Second
+	defaultDialKeepAlive = 15 * time.Second
 )
 
 // isMaxObjectSize - verify if max object size
@@ -452,7 +451,8 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout, dialKeepAlive ti
 		Proxy:                 http.ProxyFromEnvironment,
 		DialContext:           newCustomDialContext(dialTimeout, dialKeepAlive),
 		MaxIdleConnsPerHost:   256,
-		IdleConnTimeout:       60 * time.Second,
+		IdleConnTimeout:       time.Minute,
+		ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
 		TLSHandshakeTimeout:   10 * time.Second,
 		ExpectContinueTimeout: 10 * time.Second,
 		TLSClientConfig:       tlsConfig,
@@ -466,14 +466,17 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout, dialKeepAlive ti
 	}
 }
 
-// NewCustomHTTPTransport returns a new http configuration
+// NewGatewayHTTPTransport returns a new http configuration
 // used while communicating with the cloud backends.
 // This sets the value for MaxIdleConnsPerHost from 2 (go default)
 // to 256.
-func NewCustomHTTPTransport() *http.Transport {
-	return newCustomHTTPTransport(&tls.Config{
+func NewGatewayHTTPTransport() *http.Transport {
+	tr := newCustomHTTPTransport(&tls.Config{
 		RootCAs: globalRootCAs,
 	}, defaultDialTimeout, defaultDialKeepAlive)()
+	// Set aggressive timeouts for gateway
+	tr.ResponseHeaderTimeout = 30 * time.Second
+	return tr
 }
 
 // Load the json (typically from disk file).