re-use transport and set stronger backwards compatible Ciphers (#19565)

This PR fixes a few things

- FIPS support for missing for remote transports, causing
  MinIO could end up using non-FIPS Ciphers in FIPS mode

- Avoids too many transports, they all do the same thing
  to make connection pooling work properly re-use them.

- globalTCPOptions must be set before setting transport
  to make sure the client conn deadlines are honored properly.

- GCS warm tier must re-use our transport

- Re-enable trailing headers support.

cmd/admin-handlers.go

@@ -2376,7 +2376,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 	}
 
 	client := &http.Client{
-		Transport: globalHealthChkTransport,
+		Transport: globalRemoteTargetTransport,
 		Timeout:   10 * time.Second,
 	}
 

cmd/config-current.go

@@ -678,7 +678,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			}
 		}
 	case config.SubnetSubSys:
-		subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
+		subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalRemoteTargetTransport)
 		if err != nil {
 			configLogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 		} else {

cmd/endpoint.go

@@ -1205,7 +1205,7 @@ func GetProxyEndpoints(endpointServerPools EndpointServerPools) []ProxyEndpoint
 
 			proxyEps = append(proxyEps, ProxyEndpoint{
 				Endpoint:  endpoint,
-				Transport: globalProxyTransport,
+				Transport: globalRemoteTargetTransport,
 			})
 		}
 	}

cmd/globals.go

@@ -398,12 +398,8 @@ var (
 
 	globalInternodeTransport http.RoundTripper
 
-	globalProxyTransport http.RoundTripper
-
 	globalRemoteTargetTransport http.RoundTripper
 
-	globalHealthChkTransport http.RoundTripper
-
 	globalDNSCache = &dnscache.Resolver{
 		Timeout: 5 * time.Second,
 	}

cmd/server-main.go

@@ -362,22 +362,6 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 	// Initialize, see which NIC the service is running on, and save it as global value
 	setGlobalInternodeInterface(ctxt.Interface)
 
-	// allow transport to be HTTP/1.1 for proxying.
-	globalProxyTransport = NewCustomHTTPProxyTransport()()
-	globalProxyEndpoints = GetProxyEndpoints(globalEndpoints)
-	globalInternodeTransport = NewInternodeHTTPTransport(ctxt.MaxIdleConnsPerHost)()
-	globalRemoteTargetTransport = NewRemoteTargetHTTPTransport(false)()
-	globalHealthChkTransport = NewHTTPTransport()
-	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
-		PassHost:     true,
-		RoundTripper: NewHTTPTransportWithTimeout(1 * time.Hour),
-		Logger: func(err error) {
-			if err != nil && !errors.Is(err, context.Canceled) {
-				replLogIf(GlobalContext, err)
-			}
-		},
-	})
-
 	globalTCPOptions = xhttp.TCPOptions{
 		UserTimeout:        int(ctxt.UserTimeout.Milliseconds()),
 		ClientReadTimeout:  ctxt.ConnClientReadDeadline,
@@ -385,6 +369,20 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 		Interface:          ctxt.Interface,
 	}
 
+	// allow transport to be HTTP/1.1 for proxying.
+	globalProxyEndpoints = GetProxyEndpoints(globalEndpoints)
+	globalInternodeTransport = NewInternodeHTTPTransport(ctxt.MaxIdleConnsPerHost)()
+	globalRemoteTargetTransport = NewRemoteTargetHTTPTransport(false)()
+	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
+		PassHost:     true,
+		RoundTripper: globalRemoteTargetTransport,
+		Logger: func(err error) {
+			if err != nil && !errors.Is(err, context.Canceled) {
+				replLogIf(GlobalContext, err)
+			}
+		},
+	})
+
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
 	// (non-)minio process is listening on IPv4 of given port.
@@ -1024,7 +1022,7 @@ func serverMain(ctx *cli.Context) {
 		globalMinioClient, err = minio.New(globalLocalNodeName, &minio.Options{
 			Creds:     credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
 			Secure:    globalIsTLS,
-			Transport: globalProxyTransport,
+			Transport: globalRemoteTargetTransport,
 			Region:    region,
 		})
 		logger.FatalIf(err, "Unable to initialize MinIO client")

cmd/utils.go

@@ -594,29 +594,17 @@ func NewInternodeHTTPTransport(maxIdleConnsPerHost int) func() http.RoundTripper
 	}.NewInternodeHTTPTransport(maxIdleConnsPerHost)
 }
 
-// NewCustomHTTPProxyTransport is used only for proxied requests, specifically
-// only supports HTTP/1.1
-func NewCustomHTTPProxyTransport() func() *http.Transport {
-	return xhttp.ConnSettings{
-		LookupHost:       globalDNSCache.LookupHost,
-		DialTimeout:      rest.DefaultTimeout,
-		RootCAs:          globalRootCAs,
-		CipherSuites:     fips.TLSCiphers(),
-		CurvePreferences: fips.TLSCurveIDs(),
-		EnableHTTP2:      false,
-		TCPOptions:       globalTCPOptions,
-	}.NewCustomHTTPProxyTransport()
-}
-
 // NewHTTPTransportWithClientCerts returns a new http configuration
 // used while communicating with the cloud backends.
-func NewHTTPTransportWithClientCerts(clientCert, clientKey string) *http.Transport {
+func NewHTTPTransportWithClientCerts(clientCert, clientKey string) http.RoundTripper {
 	s := xhttp.ConnSettings{
-		LookupHost:  globalDNSCache.LookupHost,
+		LookupHost:       globalDNSCache.LookupHost,
-		DialTimeout: defaultDialTimeout,
+		DialTimeout:      defaultDialTimeout,
-		RootCAs:     globalRootCAs,
+		RootCAs:          globalRootCAs,
-		TCPOptions:  globalTCPOptions,
+		CipherSuites:     fips.TLSCiphersBackwardCompatible(),
-		EnableHTTP2: false,
+		CurvePreferences: fips.TLSCurveIDs(),
+		TCPOptions:       globalTCPOptions,
+		EnableHTTP2:      false,
 	}
 
 	if clientCert != "" && clientKey != "" {
@@ -633,7 +621,7 @@ func NewHTTPTransportWithClientCerts(clientCert, clientKey string) *http.Transpo
 		return transport
 	}
 
-	return s.NewHTTPTransportWithTimeout(1 * time.Minute)
+	return globalRemoteTargetTransport
 }
 
 // NewHTTPTransport returns a new http configuration
@@ -648,12 +636,14 @@ const defaultDialTimeout = 5 * time.Second
 // NewHTTPTransportWithTimeout allows setting a timeout.
 func NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport {
 	return xhttp.ConnSettings{
-		DialContext: newCustomDialContext(),
+		DialContext:      newCustomDialContext(),
-		LookupHost:  globalDNSCache.LookupHost,
+		LookupHost:       globalDNSCache.LookupHost,
-		DialTimeout: defaultDialTimeout,
+		DialTimeout:      defaultDialTimeout,
-		RootCAs:     globalRootCAs,
+		RootCAs:          globalRootCAs,
-		TCPOptions:  globalTCPOptions,
+		TCPOptions:       globalTCPOptions,
-		EnableHTTP2: false,
+		CipherSuites:     fips.TLSCiphersBackwardCompatible(),
+		CurvePreferences: fips.TLSCurveIDs(),
+		EnableHTTP2:      false,
 	}.NewHTTPTransportWithTimeout(timeout)
 }
 
@@ -682,11 +672,13 @@ func newCustomDialContext() xhttp.DialContext {
 // used while communicating with the remote replication targets.
 func NewRemoteTargetHTTPTransport(insecure bool) func() *http.Transport {
 	return xhttp.ConnSettings{
-		DialContext: newCustomDialContext(),
+		DialContext:      newCustomDialContext(),
-		LookupHost:  globalDNSCache.LookupHost,
+		LookupHost:       globalDNSCache.LookupHost,
-		RootCAs:     globalRootCAs,
+		RootCAs:          globalRootCAs,
-		TCPOptions:  globalTCPOptions,
+		CipherSuites:     fips.TLSCiphersBackwardCompatible(),
-		EnableHTTP2: false,
+		CurvePreferences: fips.TLSCurveIDs(),
+		TCPOptions:       globalTCPOptions,
+		EnableHTTP2:      false,
 	}.NewRemoteTargetHTTPTransport(insecure)
 }
 

cmd/warm-backend-gcs.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 
 	"cloud.google.com/go/storage"
 	"github.com/minio/madmin-go/v3"
@@ -102,7 +103,7 @@ func (gcs *warmBackendGCS) InUse(ctx context.Context) (bool, error) {
 	return false, nil
 }
 
-func newWarmBackendGCS(conf madmin.TierGCS, _ string) (*warmBackendGCS, error) {
+func newWarmBackendGCS(conf madmin.TierGCS, tier string) (*warmBackendGCS, error) {
 	// Validation code
 	if conf.Creds == "" {
 		return nil, errors.New("empty credentials unsupported")
@@ -117,7 +118,16 @@ func newWarmBackendGCS(conf madmin.TierGCS, _ string) (*warmBackendGCS, error) {
 		return nil, err
 	}
 
-	client, err := storage.NewClient(context.Background(), option.WithCredentialsJSON(credsJSON), option.WithScopes(storage.ScopeReadWrite))
+	clnt := &http.Client{
+		Transport: globalRemoteTargetTransport,
+	}
+
+	client, err := storage.NewClient(context.Background(),
+		option.WithCredentialsJSON(credsJSON),
+		option.WithScopes(storage.ScopeReadWrite),
+		option.WithHTTPClient(clnt),
+		option.WithUserAgent(fmt.Sprintf("gcs-tier-%s", tier)+SlashSeparator+ReleaseTag),
+	)
 	if err != nil {
 		return nil, err
 	}

cmd/warm-backend-minio.go

@@ -25,7 +25,6 @@ import (
 	"math"
 	"net/url"
 	"strings"
-	"time"
 
 	"github.com/minio/madmin-go/v3"
 	minio "github.com/minio/minio-go/v7"
@@ -108,14 +107,11 @@ func newWarmBackendMinIO(conf madmin.TierMinIO, tier string) (*warmBackendMinIO,
 	}
 
 	creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
-
-	getRemoteTierTargetInstanceTransportOnce.Do(func() {
-		getRemoteTierTargetInstanceTransport = NewHTTPTransportWithTimeout(10 * time.Minute)
-	})
 	opts := &minio.Options{
-		Creds:     creds,
+		Creds:           creds,
-		Secure:    u.Scheme == "https",
+		Secure:          u.Scheme == "https",
-		Transport: getRemoteTierTargetInstanceTransport,
+		Transport:       globalRemoteTargetTransport,
+		TrailingHeaders: true,
 	}
 	client, err := minio.New(u.Host, opts)
 	if err != nil {

cmd/warm-backend-s3.go

@@ -25,20 +25,12 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
-// getRemoteTierTargetInstanceTransport contains a singleton roundtripper.
-var (
-	getRemoteTierTargetInstanceTransport     http.RoundTripper
-	getRemoteTierTargetInstanceTransportOnce sync.Once
-)
-
 type warmBackendS3 struct {
 	client       *minio.Client
 	core         *minio.Core
@@ -162,13 +154,10 @@ func newWarmBackendS3(conf madmin.TierS3, tier string) (*warmBackendS3, error) {
 	default:
 		return nil, errors.New("insufficient parameters for S3 backend authentication")
 	}
-	getRemoteTierTargetInstanceTransportOnce.Do(func() {
-		getRemoteTierTargetInstanceTransport = NewHTTPTransportWithTimeout(10 * time.Minute)
-	})
 	opts := &minio.Options{
 		Creds:     creds,
 		Secure:    u.Scheme == "https",
-		Transport: getRemoteTierTargetInstanceTransport,
+		Transport: globalRemoteTargetTransport,
 	}
 	client, err := minio.New(u.Host, opts)
 	if err != nil {

cmd/warm-backend.go

@@ -18,11 +18,11 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/minio/madmin-go/v3"
 	xhttp "github.com/minio/minio/internal/http"
@@ -48,8 +48,7 @@ const probeObject = "probeobject"
 // checkWarmBackend checks if tier config credentials have sufficient privileges
 // to perform all operations defined in the WarmBackend interface.
 func checkWarmBackend(ctx context.Context, w WarmBackend) error {
-	var empty bytes.Reader
+	remoteVersionID, err := w.Put(ctx, probeObject, strings.NewReader("MinIO"), 5)
-	remoteVersionID, err := w.Put(ctx, probeObject, &empty, 0)
 	if err != nil {
 		if _, ok := err.(BackendDown); ok {
 			return err