simplify credentials handling in S3 gateway (#13373)

change credentials handling such that
prefer MINIO_* envs first if they work,
if not fallback to AWS credentials. If
they fail we fail to start anyways.

cmd/gateway/s3/gateway-s3-chain.go

@@ -0,0 +1,103 @@
+/*
+ * MinIO Object Storage (c) 2021 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// A Chain will search for a provider which returns credentials
+// and cache that provider until Retrieve is called again.
+//
+// The Chain provides a way of chaining multiple providers together
+// which will pick the first available using priority order of the
+// Providers in the list.
+//
+// If none of the Providers retrieve valid credentials Value, ChainProvider's
+// Retrieve() will return the no credentials value.
+//
+// If a Provider is found which returns valid credentials Value ChainProvider
+// will cache that Provider for all calls to IsExpired(), until Retrieve is
+// called again after IsExpired() is true.
+//
+//     creds := credentials.NewChainCredentials(
+//         []credentials.Provider{
+//             &credentials.EnvAWSS3{},
+//             &credentials.EnvMinio{},
+//         })
+//
+//     // Usage of ChainCredentials.
+//     mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1")
+//     if err != nil {
+//          log.Fatalln(err)
+//     }
+//
+type Chain struct {
+	Providers []credentials.Provider
+	curr      credentials.Provider
+}
+
+// NewChainCredentials returns a pointer to a new Credentials object
+// wrapping a chain of providers.
+func NewChainCredentials(providers []credentials.Provider) *credentials.Credentials {
+	for _, p := range providers {
+		if p == nil {
+			panic("providers cannot be uninitialized")
+		}
+	}
+	return credentials.New(&Chain{
+		Providers: append([]credentials.Provider{}, providers...),
+	})
+}
+
+// Retrieve returns the credentials value, returns no credentials(anonymous)
+// if no credentials provider returned any value.
+//
+// If a provider is found with credentials, it will be cached and any calls
+// to IsExpired() will return the expired state of the cached provider.
+func (c *Chain) Retrieve() (credentials.Value, error) {
+	for _, p := range c.Providers {
+		creds, _ := p.Retrieve()
+		if creds.AccessKeyID != "" && !p.IsExpired() {
+			// Only return credentials that are
+			// available and not expired.
+			c.curr = p
+			return creds, nil
+		}
+	}
+
+	providers := make([]string, 0, len(c.Providers))
+	for _, p := range c.Providers {
+		providers = append(providers, reflect.TypeOf(p).String())
+	}
+
+	return credentials.Value{}, fmt.Errorf("no credentials found in %s cannot proceed", providers)
+
+}
+
+// IsExpired will returned the expired state of the currently cached provider
+// if there is one. If there is no current provider, true will be returned.
+func (c *Chain) IsExpired() bool {
+	if c.curr != nil {
+		return c.curr.IsExpired()
+	}
+
+	return true
+}

cmd/gateway/s3/gateway-s3.go

@@ -146,7 +146,6 @@ func randString(n int, src rand.Source, prefix string) string {
 var defaultProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
-	&credentials.EnvMinio{},
 }
 
 // Chains all credential types, in the following order:
@@ -160,15 +159,17 @@ var defaultAWSCredProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
 	&credentials.IAM{
+		// you can specify a custom STS endpoint.
+		Endpoint: env.Get("MINIO_GATEWAY_S3_STS_ENDPOINT", ""),
 		Client: &http.Client{
 			Transport: minio.NewGatewayHTTPTransport(),
 		},
 	},
-	&credentials.EnvMinio{},
 }
 
-// newS3 - Initializes a new client by auto probing S3 server signature.
+// new - Initializes a new client by auto probing S3 server signature.
-func newS3(urlStr string, tripper http.RoundTripper) (*miniogo.Core, error) {
+func (g *S3) new(creds madmin.Credentials, transport http.RoundTripper) (*miniogo.Core, error) {
+	urlStr := g.host
 	if urlStr == "" {
 		urlStr = "https://s3.amazonaws.com"
 	}
@@ -184,30 +185,70 @@ func newS3(urlStr string, tripper http.RoundTripper) (*miniogo.Core, error) {
 		return nil, err
 	}
 
-	var creds *credentials.Credentials
+	var chainCreds *credentials.Credentials
 	if s3utils.IsAmazonEndpoint(*u) {
 		// If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials.
 		// Specifically IAM style rotating credentials are only supported with AWS S3 endpoint.
-		creds = credentials.NewChainCredentials(defaultAWSCredProviders)
+		chainCreds = NewChainCredentials(defaultAWSCredProviders)
-
 	} else {
-		creds = credentials.NewChainCredentials(defaultProviders)
+		chainCreds = NewChainCredentials(defaultProviders)
 	}
 
-	options := &miniogo.Options{
+	optionsStaticCreds := &miniogo.Options{
-		Creds:        creds,
+		Creds:        credentials.NewStaticV4(creds.AccessKey, creds.SecretKey, creds.SessionToken),
 		Secure:       secure,
 		Region:       s3utils.GetRegionFromURL(*u),
 		BucketLookup: miniogo.BucketLookupAuto,
-		Transport:    tripper,
+		Transport:    transport,
 	}
 
-	clnt, err := miniogo.New(endpoint, options)
+	optionsChainCreds := &miniogo.Options{
+		Creds:        chainCreds,
+		Secure:       secure,
+		Region:       s3utils.GetRegionFromURL(*u),
+		BucketLookup: miniogo.BucketLookupAuto,
+		Transport:    transport,
+	}
+
+	clntChain, err := miniogo.New(endpoint, optionsChainCreds)
 	if err != nil {
 		return nil, err
 	}
 
-	return &miniogo.Core{Client: clnt}, nil
+	clntStatic, err := miniogo.New(endpoint, optionsStaticCreds)
+	if err != nil {
+		return nil, err
+	}
+
+	if g.debug {
+		clntChain.TraceOn(os.Stderr)
+		clntStatic.TraceOn(os.Stderr)
+	}
+
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+
+	if _, err = clntStatic.BucketExists(context.Background(), probeBucketName); err != nil {
+		switch miniogo.ToErrorResponse(err).Code {
+		case "InvalidAccessKeyId":
+			// Check if the provided keys are valid for chain.
+			if _, err = clntChain.BucketExists(context.Background(), probeBucketName); err != nil {
+				if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
+					return nil, err
+				}
+			}
+			return &miniogo.Core{Client: clntChain}, nil
+		case "AccessDenied":
+			// this is a good error means backend is reachable
+			// and credentials are valid but credentials don't
+			// have access to 'probeBucketName' which is harmless.
+			return &miniogo.Core{Client: clntStatic}, nil
+		default:
+			return nil, err
+		}
+	}
+
+	// if static keys are valid always use static keys.
+	return &miniogo.Core{Client: clntStatic}, nil
 }
 
 // NewGatewayLayer returns s3 ObjectLayer.
@@ -221,24 +262,11 @@ func (g *S3) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, error
 
 	// creds are ignored here, since S3 gateway implements chaining
 	// all credentials.
-	clnt, err := newS3(g.host, t)
+	clnt, err := g.new(creds, t)
 	if err != nil {
 		return nil, err
 	}
 
-	if g.debug {
-		clnt.Client.TraceOn(os.Stderr)
-	}
-
-	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
-
-	// Check if the provided keys are valid.
-	if _, err = clnt.BucketExists(context.Background(), probeBucketName); err != nil {
-		if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
-			return nil, err
-		}
-	}
-
 	s := s3Objects{
 		Client:  clnt,
 		Metrics: metrics,
@@ -282,11 +310,17 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
 // StorageInfo is not relevant to S3 backend.
 func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []error) {
 	si.Backend.Type = madmin.Gateway
-	host := l.Client.EndpointURL().Host
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
-	if l.Client.EndpointURL().Port() == "" {
+
-		host = l.Client.EndpointURL().Host + ":" + l.Client.EndpointURL().Scheme
+	// check if bucket exists.
+	_, err := l.Client.BucketExists(ctx, probeBucketName)
+	switch miniogo.ToErrorResponse(err).Code {
+	case "", "AccessDenied":
+		si.Backend.GatewayOnline = true
+	default:
+		logger.LogIf(ctx, err)
+		si.Backend.GatewayOnline = false
 	}
-	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, host)
 	return si, nil
 }
 

cmd/healthcheck-handler.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"strconv"
 
@@ -29,6 +30,11 @@ const unavailable = "offline"
 
 // ClusterCheckHandler returns if the server is ready for requests.
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if globalIsGateway {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
 	ctx := newContext(r, w, "ClusterCheckHandler")
 
 	if shouldProxy() {
@@ -67,6 +73,11 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 // ClusterReadCheckHandler returns if the server is ready for requests.
 func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if globalIsGateway {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
 	ctx := newContext(r, w, "ClusterReadCheckHandler")
 
 	if shouldProxy() {
@@ -85,28 +96,13 @@ func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
+
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
 // ReadinessCheckHandler Checks if the process is up. Always returns success.
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	if shouldProxy() {
+	LivenessCheckHandler(w, r)
-		// Service not initialized yet
-		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
-	}
-
-	if globalIsGateway && globalEtcdClient != nil {
-		// Borrowed from https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118
-		ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout)
-		defer cancel()
-		// etcd unreachable throw an error for readiness.
-		if _, err := globalEtcdClient.Get(ctx, "health"); err != nil {
-			writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
-			return
-		}
-	}
-
-	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
@@ -116,15 +112,49 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 	}
 
-	if globalIsGateway && globalEtcdClient != nil {
+	if globalIsGateway {
-		// Borrowed from https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118
+		objLayer := newObjectLayerFn()
-		ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout)
+		if objLayer == nil {
-		defer cancel()
+			apiErr := toAPIError(r.Context(), errServerNotInitialized)
-		// etcd unreachable throw an error for readiness.
+			switch r.Method {
-		if _, err := globalEtcdClient.Get(ctx, "health"); err != nil {
+			case http.MethodHead:
-			writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, apiErr, r.URL)
+			}
 			return
 		}
+
+		storageInfo, _ := objLayer.StorageInfo(r.Context())
+		if !storageInfo.Backend.GatewayOnline {
+			err := errors.New("gateway backend is not reachable")
+			apiErr := toAPIError(r.Context(), err)
+			switch r.Method {
+			case http.MethodHead:
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, apiErr, r.URL)
+			}
+			return
+		}
+
+		if globalEtcdClient != nil {
+			// Borrowed from
+			// https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118
+			ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout)
+			defer cancel()
+			if _, err := globalEtcdClient.Get(ctx, "health"); err != nil {
+				// etcd unreachable throw an error..
+				switch r.Method {
+				case http.MethodHead:
+					apiErr := toAPIError(r.Context(), err)
+					writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+				case http.MethodGet:
+					writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+				}
+				return
+			}
+		}
 	}
 
 	writeResponse(w, http.StatusOK, nil, mimeNone)