mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
tiering: add aws role support for s3 (#12424)
Signed-off-by: Poorna Krishnamoorthy <poorna@minio.io>
This commit is contained in:
parent
36b2f6d11d
commit
f199afcd6c
@ -27,6 +27,7 @@ import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@ -39,6 +40,7 @@ import (
|
||||
dns2 "github.com/miekg/dns"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/kes"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
@ -54,6 +56,7 @@ import (
|
||||
|
||||
// serverDebugLog will enable debug printing
|
||||
var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
|
||||
var defaultAWSCredProvider []credentials.Provider
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UTC().UnixNano())
|
||||
@ -74,7 +77,6 @@ func init() {
|
||||
// safe to assume a higher timeout upto 10 minutes.
|
||||
globalDNSCache = xhttp.NewDNSCache(10*time.Minute, 5*time.Second, logger.LogOnceIf)
|
||||
}
|
||||
|
||||
initGlobalContext()
|
||||
|
||||
globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
|
||||
@ -92,6 +94,14 @@ func init() {
|
||||
console.SetColor("Debug", color.New())
|
||||
|
||||
gob.Register(StorageErr(""))
|
||||
|
||||
defaultAWSCredProvider = []credentials.Provider{
|
||||
&credentials.IAM{
|
||||
Client: &http.Client{
|
||||
Transport: NewGatewayHTTPTransport(),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
|
||||
|
12
cmd/tier.go
12
cmd/tier.go
@ -144,12 +144,16 @@ func (config *TierConfigMgr) Edit(ctx context.Context, tierName string, creds ma
|
||||
newCfg := config.Tiers[tierName]
|
||||
switch tierType {
|
||||
case madmin.S3:
|
||||
if creds.AccessKey == "" || creds.SecretKey == "" {
|
||||
if (creds.AccessKey == "" || creds.SecretKey == "") && !creds.AWSRole {
|
||||
return errTierInsufficientCreds
|
||||
}
|
||||
newCfg.S3.AccessKey = creds.AccessKey
|
||||
newCfg.S3.SecretKey = creds.SecretKey
|
||||
|
||||
switch {
|
||||
case creds.AWSRole:
|
||||
newCfg.S3.AWSRole = true
|
||||
default:
|
||||
newCfg.S3.AccessKey = creds.AccessKey
|
||||
newCfg.S3.SecretKey = creds.SecretKey
|
||||
}
|
||||
case madmin.Azure:
|
||||
if creds.AccessKey == "" || creds.SecretKey == "" {
|
||||
return errTierInsufficientCreds
|
||||
|
@ -106,7 +106,12 @@ func newWarmBackendS3(conf madmin.TierS3) (*warmBackendS3, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
|
||||
var creds *credentials.Credentials
|
||||
if conf.AWSRole {
|
||||
creds = credentials.NewChainCredentials(defaultAWSCredProvider)
|
||||
} else {
|
||||
creds = credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
|
||||
}
|
||||
getRemoteTargetInstanceTransportOnce.Do(func() {
|
||||
getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
|
||||
})
|
||||
|
@ -116,6 +116,11 @@ Using above tier, set up a lifecycle rule with transition:
|
||||
mc ilm add --expiry-days 365 --transition-days 45 --storage-class "AZURETIER" myminio/srcbucket
|
||||
```
|
||||
|
||||
Note: In the case of S3, it is possible to create a tier from MinIO running in EC2 to S3 using AWS role attached to EC2 as credentials instead of accesskey/secretkey:
|
||||
```
|
||||
mc admin tier add s3 source S3TIER --bucket s3bucket --prefix testprefix/ --use-aws-role
|
||||
```
|
||||
|
||||
Once transitioned, GET or HEAD on the object will stream the content from the transitioned tier. In the event that the object needs to be restored temporarily to the local cluster, the AWS [RestoreObject API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html) can be utilized.
|
||||
|
||||
```
|
||||
|
2
go.mod
2
go.mod
@ -44,7 +44,7 @@ require (
|
||||
github.com/minio/csvparser v1.0.0
|
||||
github.com/minio/highwayhash v1.0.2
|
||||
github.com/minio/kes v0.14.0
|
||||
github.com/minio/madmin-go v1.0.9
|
||||
github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4
|
||||
github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78
|
||||
github.com/minio/parquet-go v1.0.0
|
||||
github.com/minio/pkg v1.0.4
|
||||
|
6
go.sum
6
go.sum
@ -483,8 +483,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
|
||||
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
|
||||
github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q=
|
||||
github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA=
|
||||
github.com/minio/madmin-go v1.0.9 h1:zXZMppnqboIyELPirHcU6qxrnJkVwj2k7rLIB0T12sY=
|
||||
github.com/minio/madmin-go v1.0.9/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
|
||||
github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4 h1:AxtnO3AODg0t2IPXbrqmDBhGZTcrUhlT/ixdLQQ3164=
|
||||
github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
|
||||
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
|
||||
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
|
||||
github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
|
||||
@ -492,8 +492,6 @@ github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 h1:v7OMbUnWky
|
||||
github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
|
||||
github.com/minio/parquet-go v1.0.0 h1:fcWsEvub04Nsl/4hiRBDWlbqd6jhacQieV07a+nhiIk=
|
||||
github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTacxxk/oHA=
|
||||
github.com/minio/pkg v1.0.3 h1:tUhM6lG/BdNB0+5f2RbE4ifCAYwMs6cRJnZ/AY0WIeQ=
|
||||
github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
|
||||
github.com/minio/pkg v1.0.4 h1:+BmaCENP6BaMm9PsGK6L1L5MKulWDxl4qobvJYf6m/E=
|
||||
github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
|
||||
github.com/minio/rpc v1.0.0 h1:tJCHyLfQF6k6HlMQFpKy2FO/7lc2WP8gLDGMZp18E70=
|
||||
|
Loading…
Reference in New Issue
Block a user