diff --git a/cmd/bucket-listobjects-handlers.go b/cmd/bucket-listobjects-handlers.go index 567864d30..697e65c98 100644 --- a/cmd/bucket-listobjects-handlers.go +++ b/cmd/bucket-listobjects-handlers.go @@ -17,6 +17,8 @@ package cmd import ( + "context" + "io" "net/http" "strings" @@ -157,6 +159,10 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt return } + if proxyListRequest(ctx, w, r, bucket) { + return + } + listObjectsV2 := objectAPI.ListObjectsV2 // Inititate a list objects operation based on the input params. @@ -231,6 +237,10 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http return } + if proxyListRequest(ctx, w, r, bucket) { + return + } + listObjectsV2 := objectAPI.ListObjectsV2 // Inititate a list objects operation based on the input params. @@ -262,6 +272,43 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http writeSuccessResponseXML(w, encodeResponse(response)) } +func getListEndpoint(bucket string) ListEndpoint { + return globalListEndpoints[crcHashMod(bucket, len(globalListEndpoints))] +} + +// Proxy the list request to the right server. +func proxyListRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, bucket string) (success bool) { + if len(globalListEndpoints) == 0 { + return false + } + ep := getListEndpoint(bucket) + if ep.isLocal { + return false + } + ctx = r.Context() + outreq := r.Clone(ctx) + outreq.URL.Scheme = "http" + outreq.URL.Host = ep.host + outreq.URL.Path = r.URL.Path + outreq.Header.Add("Host", r.Host) + if globalIsSSL { + outreq.URL.Scheme = "https" + } + outreq.Host = r.Host + res, err := ep.t.RoundTrip(outreq) + if err != nil { + return false + } + for k, vv := range res.Header { + for _, v := range vv { + w.Header().Set(k, v) + } + } + w.WriteHeader(res.StatusCode) + io.Copy(w, res.Body) + return true +} + // ListObjectsV1Handler - GET Bucket (List Objects) Version 1. // -------------------------- // This implementation of the GET operation returns some or all (up to 10000) @@ -300,6 +347,10 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http return } + if proxyListRequest(ctx, w, r, bucket) { + return + } + listObjects := objectAPI.ListObjects // Inititate a list objects operation based on the input params. diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 8dabc2022..79d2adc20 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -17,8 +17,10 @@ package cmd import ( + "crypto/tls" "fmt" "net" + "net/http" "net/url" "path" "path/filepath" @@ -31,6 +33,7 @@ import ( "github.com/minio/minio-go/v6/pkg/set" "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/cmd/rest" "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/mountinfo" ) @@ -46,6 +49,14 @@ const ( URLEndpointType ) +// ListEndpoint - endpoint used for list redirects +// See proxyListRequest() for details. +type ListEndpoint struct { + host string + t *http.Transport + isLocal bool +} + // Endpoint - any type of endpoint. type Endpoint struct { *url.URL @@ -708,6 +719,50 @@ func GetRemotePeers(endpointZones EndpointZones) []string { return peerSet.ToSlice() } +// GetListEndpoints - get all endpoints that can be used to proxy list request. +func GetListEndpoints(endpointZones EndpointZones) ([]ListEndpoint, error) { + var listeps []ListEndpoint + + listepExists := func(host string) bool { + for _, listep := range listeps { + if listep.host == host { + return true + } + } + return false + } + + for _, ep := range endpointZones { + for _, endpoint := range ep.Endpoints { + if endpoint.Type() != URLEndpointType { + continue + } + + host := endpoint.Host + if listepExists(host) { + continue + } + hostName, _, err := net.SplitHostPort(host) + if err != nil { + return nil, err + } + var tlsConfig *tls.Config + if globalIsSSL { + tlsConfig = &tls.Config{ + ServerName: hostName, + RootCAs: globalRootCAs, + } + } + listeps = append(listeps, ListEndpoint{ + host, + newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)(), + endpoint.IsLocal, + }) + } + } + return listeps, nil +} + func updateDomainIPs(endPoints set.StringSet) { ipList := set.NewStringSet() for e := range endPoints { diff --git a/cmd/globals.go b/cmd/globals.go index 6fa21d237..87e7a3568 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -279,6 +279,7 @@ var ( // If writes to FS backend should be O_SYNC. globalFSOSync bool + globalListEndpoints []ListEndpoint // Add new variable global values here. ) diff --git a/cmd/server-main.go b/cmd/server-main.go index c04425413..6eafca93a 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -396,6 +396,9 @@ func serverMain(ctx *cli.Context) { globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get()) logger.FatalIf(err, "Failed to read root CAs (%v)", err) + globalListEndpoints, err = GetListEndpoints(globalEndpoints) + logger.FatalIf(err, "Invalid command line arguments") + globalMinioEndpoint = func() string { host := globalMinioHost if host == "" {