tiering: add aws role support for s3 (#12424)

Signed-off-by: Poorna Krishnamoorthy <poorna@minio.io>

cmd/common-main.go

@@ -27,6 +27,7 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -39,6 +40,7 @@ import (
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/kes"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
@@ -54,6 +56,7 @@ import (
 
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+var defaultAWSCredProvider []credentials.Provider
 
 func init() {
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -74,7 +77,6 @@ func init() {
 		// safe to assume a higher timeout upto 10 minutes.
 		globalDNSCache = xhttp.NewDNSCache(10*time.Minute, 5*time.Second, logger.LogOnceIf)
 	}
-
 	initGlobalContext()
 
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
@@ -92,6 +94,14 @@ func init() {
 	console.SetColor("Debug", color.New())
 
 	gob.Register(StorageErr(""))
+
+	defaultAWSCredProvider = []credentials.Provider{
+		&credentials.IAM{
+			Client: &http.Client{
+				Transport: NewGatewayHTTPTransport(),
+			},
+		},
+	}
 }
 
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {

cmd/tier.go

@@ -144,12 +144,16 @@ func (config *TierConfigMgr) Edit(ctx context.Context, tierName string, creds ma
 	newCfg := config.Tiers[tierName]
 	switch tierType {
 	case madmin.S3:
-		if creds.AccessKey == "" || creds.SecretKey == "" {
+		if (creds.AccessKey == "" || creds.SecretKey == "") && !creds.AWSRole {
 			return errTierInsufficientCreds
 		}
+		switch {
+		case creds.AWSRole:
+			newCfg.S3.AWSRole = true
+		default:
 			newCfg.S3.AccessKey = creds.AccessKey
 			newCfg.S3.SecretKey = creds.SecretKey
-
+		}
 	case madmin.Azure:
 		if creds.AccessKey == "" || creds.SecretKey == "" {
 			return errTierInsufficientCreds

cmd/warm-backend-s3.go

@@ -106,7 +106,12 @@ func newWarmBackendS3(conf madmin.TierS3) (*warmBackendS3, error) {
 	if err != nil {
 		return nil, err
 	}
-	creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
+	var creds *credentials.Credentials
+	if conf.AWSRole {
+		creds = credentials.NewChainCredentials(defaultAWSCredProvider)
+	} else {
+		creds = credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
+	}
 	getRemoteTargetInstanceTransportOnce.Do(func() {
 		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
 	})

docs/bucket/lifecycle/README.md

@@ -116,6 +116,11 @@ Using above tier, set up a lifecycle rule with transition:
  mc ilm add --expiry-days 365 --transition-days 45 --storage-class "AZURETIER" myminio/srcbucket
 ```
 
+Note: In the case of S3, it is possible to create a tier from MinIO running in EC2 to S3 using AWS role attached to EC2 as credentials instead of accesskey/secretkey:
+```
+mc admin tier add s3 source S3TIER --bucket s3bucket --prefix testprefix/ --use-aws-role
+```
+
 Once transitioned, GET or HEAD on the object will stream the content from the transitioned tier. In the event that the object needs to be restored temporarily to the local cluster, the AWS [RestoreObject API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html) can be utilized.
 
 ```

go.mod

@@ -44,7 +44,7 @@ require (
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.0.9
+	github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4
 	github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78
 	github.com/minio/parquet-go v1.0.0
 	github.com/minio/pkg v1.0.4

go.sum

@@ -483,8 +483,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q=
 github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA=
-github.com/minio/madmin-go v1.0.9 h1:zXZMppnqboIyELPirHcU6qxrnJkVwj2k7rLIB0T12sY=
+github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4 h1:AxtnO3AODg0t2IPXbrqmDBhGZTcrUhlT/ixdLQQ3164=
-github.com/minio/madmin-go v1.0.9/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
+github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
 github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
 github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
@@ -492,8 +492,6 @@ github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 h1:v7OMbUnWky
 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
 github.com/minio/parquet-go v1.0.0 h1:fcWsEvub04Nsl/4hiRBDWlbqd6jhacQieV07a+nhiIk=
 github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTacxxk/oHA=
-github.com/minio/pkg v1.0.3 h1:tUhM6lG/BdNB0+5f2RbE4ifCAYwMs6cRJnZ/AY0WIeQ=
-github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.4 h1:+BmaCENP6BaMm9PsGK6L1L5MKulWDxl4qobvJYf6m/E=
 github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/rpc v1.0.0 h1:tJCHyLfQF6k6HlMQFpKy2FO/7lc2WP8gLDGMZp18E70=