Allow minio s3 gateway to use different AWS auth mechanisms (#6422)

Allow minio s3 gateway to use aws environment credentials,
IAM instance credentials, or AWS file credentials.

If AWS_ACCESS_KEY_ID, AWS_SECRET_ACCSES_KEY are set, 
or minio is running on an ec2 instance with IAM instance credentials, 
or there is a file $HOME/.aws/credentials, minio running as an S3
gateway will authenticate with AWS S3 using those one of credentials.

The lookup order:
1. AWS environment varaibles
2. IAM instance credentials
3. $HOME/.aws/credentials
4. minio environment variables

To authenticate with the minio gateway, you will always use the
minio environment variables MINIO_ACCESS_KEY MINIO_SECRET_KEY.

cmd/gateway/s3/gateway-s3.go

@@ -21,11 +21,13 @@ import (
 	"encoding/json"
 	"io"
 	"math/rand"
+	"net/http"
 	"strings"
 	"time"
 
 	"github.com/minio/cli"
 	miniogo "github.com/minio/minio-go"
+	"github.com/minio/minio-go/pkg/credentials"
 	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -88,6 +90,15 @@ EXAMPLES:
      $ export MINIO_CACHE_EXPIRY=40
      $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
+
+  4. Start minio gateway server for AWS S3 backend using AWS environment variables.
+     NOTE: The access and secret key in this case will authenticate with Minio instead
+     of AWS and AWS envs will be used to authenticate to AWS S3.
+     $ export AWS_ACCESS_KEY_ID=aws_access_key
+     $ export AWS_SECRET_ACCESS_KEY=aws_secret_key
+     $ export MINIO_ACCESS_KEY=accesskey
+     $ export MINIO_SECRET_KEY=secretkey
+     $ {{.HelpName}}
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
@@ -149,7 +160,7 @@ func randString(n int, src rand.Source, prefix string) string {
 }
 
 // newS3 - Initializes a new client by auto probing S3 server signature.
-func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
+func newS3(url string) (*miniogo.Core, error) {
 	if url == "" {
 		url = "https://s3.amazonaws.com"
 	}
@@ -160,19 +171,33 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 		return nil, err
 	}
 
-	clnt, err := miniogo.NewV4(endpoint, accessKey, secretKey, secure)
+	// Chains all credential types, in the following order:
+	//  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
+	//  - IAM profile based credentials. (performs an HTTP
+	//    call to a pre-defined endpoint, only valid inside
+	//    configured ec2 instances)
+	//  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
+	//  - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
+	creds := credentials.NewChainCredentials([]credentials.Provider{
+		&credentials.EnvAWS{},
+		&credentials.IAM{
+			Client: &http.Client{
+				Transport: minio.NewCustomHTTPTransport(),
+			},
+		},
+		&credentials.FileAWSCredentials{},
+		&credentials.EnvMinio{},
+	})
+
+	clnt, err := miniogo.NewWithCredentials(endpoint, creds, secure, "")
 	if err != nil {
 		return nil, err
 	}
+
 	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+	// Check if the provided keys are valid.
 	if _, err = clnt.BucketExists(probeBucketName); err != nil {
-		clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
+		return nil, err
-		if err != nil {
-			return nil, err
-		}
-		if _, err = clnt.BucketExists(probeBucketName); err != nil {
-			return nil, err
-		}
 	}
 
 	return &miniogo.Core{Client: clnt}, nil
@@ -180,8 +205,9 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 
 // NewGatewayLayer returns s3 ObjectLayer.
 func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	// Probe S3 signature with input credentials.
+	// creds are ignored here, since S3 gateway implements chaining
-	clnt, err := newS3(g.host, creds.AccessKey, creds.SecretKey)
+	// all credentials.
+	clnt, err := newS3(g.host)
 	if err != nil {
 		return nil, err
 	}

docs/gateway/s3.md

@@ -3,11 +3,9 @@
 Minio S3 Gateway adds Minio features like Minio Browser and disk caching to AWS S3 or any other AWS S3 compatible service.
 
 ## Run Minio Gateway for AWS S3
-
+As a prerequisite to run Minio S3 gateway, you need valid AWS S3 access key and secret key by default. Optionally you can also set custom access/secret key, when you have rotating AWS IAM credentials or AWS credentials through environment variables (i.e. AWS_ACCESS_KEY_ID)
-As a prerequisite to run Minio S3 gateway, you need valid AWS S3 access key and secret key.
 
 ### Using Docker
-
 ```
 docker run -p 9000:9000 --name minio-s3 \
  -e "MINIO_ACCESS_KEY=aws_s3_access_key" \
@@ -16,19 +14,30 @@ docker run -p 9000:9000 --name minio-s3 \
 ```
 
 ### Using Binary
-
 ```
 export MINIO_ACCESS_KEY=aws_s3_access_key
 export MINIO_SECRET_KEY=aws_s3_secret_key
 minio gateway s3
 ```
 
-## Run Minio Gateway for AWS S3 compatible services
+### Using Binary in EC2
+Using IAM rotating credentials for AWS S3
+```
+export MINIO_ACCESS_KEY=custom_access_key
+export MINIO_SECRET_KEY=custom_secret_key
+minio gateway s3
+```
 
+Minio gateway will automatically look for list of credential styles in following order.
+
+- AWS env vars (i.e. AWS_ACCESS_KEY_ID)
+- IAM profile based credentials. (performs an HTTP call to a pre-defined endpoint, only valid inside configured ec2 instances)
+- AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
+
+## Run Minio Gateway for AWS S3 compatible services
 As a prerequisite to run Minio S3 gateway on an AWS S3 compatible service, you need valid access key, secret key and service endpoint.
 
 ### Using Docker
-
 ```
 docker run -p 9000:9000 --name minio-s3 \
  -e "MINIO_ACCESS_KEY=access_key" \
@@ -37,7 +46,6 @@ docker run -p 9000:9000 --name minio-s3 \
 ```
 
 ### Using Binary
-
 ```
 export MINIO_ACCESS_KEY=access_key
 export MINIO_SECRET_KEY=secret_key
@@ -45,7 +53,6 @@ minio gateway s3 https://s3_compatible_service_endpoint:port
 ```
 
 ## Minio Caching
-
 Minio edge caching allows storing content closer to the applications. Frequently accessed objects are stored in a local disk based cache. Edge caching with Minio gateway feature allows
 
 - Dramatic improvements for time to first byte for any object.
@@ -54,7 +61,6 @@ Minio edge caching allows storing content closer to the applications. Frequently
 Refer [this document](https://docs.minio.io/docs/minio-disk-cache-guide.html) to get started with Minio Caching.
 
 ## Minio Browser
-
 Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully.
 
 ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser-gateway.png?raw=true)