simplify credentials handling in S3 gateway (#13373)

change credentials handling such that
prefer MINIO_* envs first if they work,
if not fallback to AWS credentials. If
they fail we fail to start anyways.

cmd/gateway/s3/gateway-s3.go

@@ -146,7 +146,6 @@ func randString(n int, src rand.Source, prefix string) string {
 var defaultProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
-	&credentials.EnvMinio{},
 }
 
 // Chains all credential types, in the following order:
@@ -160,15 +159,17 @@ var defaultAWSCredProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
 	&credentials.IAM{
+		// you can specify a custom STS endpoint.
+		Endpoint: env.Get("MINIO_GATEWAY_S3_STS_ENDPOINT", ""),
 		Client: &http.Client{
 			Transport: minio.NewGatewayHTTPTransport(),
 		},
 	},
-	&credentials.EnvMinio{},
 }
 
-// newS3 - Initializes a new client by auto probing S3 server signature.
-func newS3(urlStr string, tripper http.RoundTripper) (*miniogo.Core, error) {
+// new - Initializes a new client by auto probing S3 server signature.
+func (g *S3) new(creds madmin.Credentials, transport http.RoundTripper) (*miniogo.Core, error) {
+	urlStr := g.host
 	if urlStr == "" {
 		urlStr = "https://s3.amazonaws.com"
 	}
@@ -184,30 +185,70 @@ func newS3(urlStr string, tripper http.RoundTripper) (*miniogo.Core, error) {
 		return nil, err
 	}
 
-	var creds *credentials.Credentials
+	var chainCreds *credentials.Credentials
 	if s3utils.IsAmazonEndpoint(*u) {
 		// If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials.
 		// Specifically IAM style rotating credentials are only supported with AWS S3 endpoint.
-		creds = credentials.NewChainCredentials(defaultAWSCredProviders)
-
+		chainCreds = NewChainCredentials(defaultAWSCredProviders)
 	} else {
-		creds = credentials.NewChainCredentials(defaultProviders)
+		chainCreds = NewChainCredentials(defaultProviders)
 	}
 
-	options := &miniogo.Options{
-		Creds:        creds,
+	optionsStaticCreds := &miniogo.Options{
+		Creds:        credentials.NewStaticV4(creds.AccessKey, creds.SecretKey, creds.SessionToken),
 		Secure:       secure,
 		Region:       s3utils.GetRegionFromURL(*u),
 		BucketLookup: miniogo.BucketLookupAuto,
-		Transport:    tripper,
+		Transport:    transport,
 	}
 
-	clnt, err := miniogo.New(endpoint, options)
+	optionsChainCreds := &miniogo.Options{
+		Creds:        chainCreds,
+		Secure:       secure,
+		Region:       s3utils.GetRegionFromURL(*u),
+		BucketLookup: miniogo.BucketLookupAuto,
+		Transport:    transport,
+	}
+
+	clntChain, err := miniogo.New(endpoint, optionsChainCreds)
 	if err != nil {
 		return nil, err
 	}
 
-	return &miniogo.Core{Client: clnt}, nil
+	clntStatic, err := miniogo.New(endpoint, optionsStaticCreds)
+	if err != nil {
+		return nil, err
+	}
+
+	if g.debug {
+		clntChain.TraceOn(os.Stderr)
+		clntStatic.TraceOn(os.Stderr)
+	}
+
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+
+	if _, err = clntStatic.BucketExists(context.Background(), probeBucketName); err != nil {
+		switch miniogo.ToErrorResponse(err).Code {
+		case "InvalidAccessKeyId":
+			// Check if the provided keys are valid for chain.
+			if _, err = clntChain.BucketExists(context.Background(), probeBucketName); err != nil {
+				if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
+					return nil, err
+				}
+			}
+			return &miniogo.Core{Client: clntChain}, nil
+		case "AccessDenied":
+			// this is a good error means backend is reachable
+			// and credentials are valid but credentials don't
+			// have access to 'probeBucketName' which is harmless.
+			return &miniogo.Core{Client: clntStatic}, nil
+		default:
+			return nil, err
+		}
+	}
+
+	// if static keys are valid always use static keys.
+	return &miniogo.Core{Client: clntStatic}, nil
 }
 
 // NewGatewayLayer returns s3 ObjectLayer.
@@ -221,24 +262,11 @@ func (g *S3) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, error
 
 	// creds are ignored here, since S3 gateway implements chaining
 	// all credentials.
-	clnt, err := newS3(g.host, t)
+	clnt, err := g.new(creds, t)
 	if err != nil {
 		return nil, err
 	}
 
-	if g.debug {
-		clnt.Client.TraceOn(os.Stderr)
-	}
-
-	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
-
-	// Check if the provided keys are valid.
-	if _, err = clnt.BucketExists(context.Background(), probeBucketName); err != nil {
-		if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
-			return nil, err
-		}
-	}
-
 	s := s3Objects{
 		Client:  clnt,
 		Metrics: metrics,
@@ -282,11 +310,17 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
 // StorageInfo is not relevant to S3 backend.
 func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []error) {
 	si.Backend.Type = madmin.Gateway
-	host := l.Client.EndpointURL().Host
-	if l.Client.EndpointURL().Port() == "" {
-		host = l.Client.EndpointURL().Host + ":" + l.Client.EndpointURL().Scheme
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+
+	// check if bucket exists.
+	_, err := l.Client.BucketExists(ctx, probeBucketName)
+	switch miniogo.ToErrorResponse(err).Code {
+	case "", "AccessDenied":
+		si.Backend.GatewayOnline = true
+	default:
+		logger.LogIf(ctx, err)
+		si.Backend.GatewayOnline = false
 	}
-	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, host)
 	return si, nil
 }