MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Support for remote tier management (#12090)

Browse Source 
With this change, MinIO's ILM supports transitioning objects to a remote tier.
This change includes support for Azure Blob Storage, AWS S3 compatible object
storage incl. MinIO and Google Cloud Storage as remote tier storage backends.

Some new additions include:

 - Admin APIs remote tier configuration management

 - Simple journal to track remote objects to be 'collected'
   This is used by object API handlers which 'mutate' object versions by
   overwriting/replacing content (Put/CopyObject) or removing the version
   itself (e.g DeleteObjectVersion).

 - Rework of previous ILM transition to fit the new model
   In the new model, a storage class (a.k.a remote tier) is defined by the
   'remote' object storage type (one of s3, azure, GCS), bucket name and a
   prefix.

* Fixed bugs, review comments, and more unit-tests

- Leverage inline small object feature
- Migrate legacy objects to the latest object format before transitioning
- Fix restore to particular version if specified
- Extend SharedDataDirCount to handle transitioned and restored objects
- Restore-object should accept version-id for version-suspended bucket (#12091)
- Check if remote tier creds have sufficient permissions
- Bonus minor fixes to existing error messages

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Krishna Srinivas <krishna@minio.io>
Signed-off-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Krishnan Parthasarathi
2021-04-19 10:30:42 -07:00
committed by Harshavardhana
parent 069432566f
commit c829e3a13b
302 changed files with 10260 additions and 3800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
cmd/bucket-targets.go
View File

@@ -23,7 +23,6 @@ import (
"encoding/hex"
"encoding/json"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
@@ -122,45 +121,24 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
return NotImplemented{Message: "Synchronous replication does not support bandwidth limits"}
}
}
if tgt.Type == madmin.ILMService {
if globalBucketVersioningSys.Enabled(bucket) {
vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
if err != nil {
if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
}
return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
}
if vcfg.Status != string(versioning.Enabled) {
return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
}
}
}
sys.Lock()
defer sys.Unlock()
tgts := sys.targetsMap[bucket]
newtgts := make([]madmin.BucketTarget, len(tgts))
labels := make(map[string]struct{}, len(tgts))
found := false
for idx, t := range tgts {
labels[t.Label] = struct{}{}
if t.Type == tgt.Type {
if t.Arn == tgt.Arn && !update {
return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
}
if t.Label == tgt.Label && !update {
return BucketRemoteLabelInUse{Bucket: t.TargetBucket}
}
newtgts[idx] = *tgt
found = true
continue
}
newtgts[idx] = t
}
if _, ok := labels[tgt.Label]; ok && !update {
return BucketRemoteLabelInUse{Bucket: tgt.TargetBucket}
}
if !found && !update {
newtgts = append(newtgts, *tgt)
}
@@ -194,15 +172,6 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
}
}
}
if arn.Type == madmin.ILMService {
// reject removal of remote target if lifecycle transition uses this arn
config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
if err == nil && transitionSCInUse(ctx, config, bucket, arnStr) {
if _, ok := sys.arnRemotesMap[arnStr]; ok {
return BucketRemoteRemoveDisallowed{Bucket: bucket}
}
}
}
// delete ARN type from list of matching targets
sys.Lock()
@@ -235,44 +204,6 @@ func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn strin
return sys.arnRemotesMap[arn]
}
// GetRemoteTargetWithLabel returns bucket target given a target label
func (sys *BucketTargetSys) GetRemoteTargetWithLabel(ctx context.Context, bucket, targetLabel string) *madmin.BucketTarget {
sys.RLock()
defer sys.RUnlock()
for _, t := range sys.targetsMap[bucket] {
if strings.ToUpper(t.Label) == strings.ToUpper(targetLabel) {
tgt := t.Clone()
return &tgt
}
}
return nil
}
// GetRemoteArnWithLabel returns bucket target's ARN given its target label
func (sys *BucketTargetSys) GetRemoteArnWithLabel(ctx context.Context, bucket, tgtLabel string) *madmin.ARN {
tgt := sys.GetRemoteTargetWithLabel(ctx, bucket, tgtLabel)
if tgt == nil {
return nil
}
arn, err := madmin.ParseARN(tgt.Arn)
if err != nil {
return nil
}
return arn
}
// GetRemoteLabelWithArn returns a bucket target's label given its ARN
func (sys *BucketTargetSys) GetRemoteLabelWithArn(ctx context.Context, bucket, arnStr string) string {
sys.RLock()
defer sys.RUnlock()
for _, t := range sys.targetsMap[bucket] {
if t.Arn == arnStr {
return t.Label
}
}
return ""
}
// NewBucketTargetSys - creates new replication system.
func NewBucketTargetSys() *BucketTargetSys {
return &BucketTargetSys{
@@ -362,7 +293,6 @@ var getRemoteTargetInstanceTransportOnce sync.Once
func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) {
config := tcfg.Credentials
creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
getRemoteTargetInstanceTransportOnce.Do(func() {
getRemoteTargetInstanceTransport = NewRemoteTargetHTTPTransport()
})
@@ -382,8 +312,9 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
tc := &TargetClient{
Client: api,
healthCheckDuration: hcDuration,
bucket: tcfg.TargetBucket,
replicateSync: tcfg.ReplicationSync,
Bucket: tcfg.TargetBucket,
StorageClass: tcfg.StorageClass,
}
go tc.healthCheck()
return tc, nil
@@ -459,8 +390,9 @@ type TargetClient struct {
*miniogo.Client
up int32
healthCheckDuration time.Duration
bucket string // remote bucket target
Bucket string // remote bucket target
replicateSync bool
StorageClass string // storage class on remote
}
func (tc *TargetClient) isOffline() bool {
@@ -469,7 +401,7 @@ func (tc *TargetClient) isOffline() bool {
func (tc *TargetClient) healthCheck() {
for {
_, err := tc.BucketExists(GlobalContext, tc.bucket)
_, err := tc.BucketExists(GlobalContext, tc.Bucket)
if err != nil {
atomic.StoreInt32(&tc.up, 0)
time.Sleep(tc.healthCheckDuration)