fix: Change ListBucketTargets handler (#10217)

to list all targets across a tenant.
Also fixing some validations.
This commit is contained in:
poornas 2020-08-06 17:10:21 -07:00 committed by GitHub
parent ce129efa09
commit adcaa6f9de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 180 additions and 85 deletions

View File

@ -26,7 +26,6 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/env"
iampolicy "github.com/minio/minio/pkg/iam/policy" iampolicy "github.com/minio/minio/pkg/iam/policy"
"github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/madmin"
@ -122,8 +121,8 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
writeSuccessResponseJSON(w, configData) writeSuccessResponseJSON(w, configData)
} }
// SetBucketTargetHandler - sets a remote target for bucket // SetRemoteTargetHandler - sets a remote target for bucket
func (a adminAPIHandlers) SetBucketTargetHandler(w http.ResponseWriter, r *http.Request) { func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "SetBucketTarget") ctx := newContext(r, w, "SetBucketTarget")
defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r)) defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
@ -174,6 +173,7 @@ func (a adminAPIHandlers) SetBucketTargetHandler(w http.ResponseWriter, r *http.
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL) writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
return return
} }
target.SourceBucket = bucket
target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target) target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
if target.Arn == "" { if target.Arn == "" {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
@ -183,7 +183,7 @@ func (a adminAPIHandlers) SetBucketTargetHandler(w http.ResponseWriter, r *http.
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return return
} }
targets, err := globalBucketTargetSys.ListTargets(ctx, bucket) targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
@ -209,9 +209,9 @@ func (a adminAPIHandlers) SetBucketTargetHandler(w http.ResponseWriter, r *http.
writeSuccessResponseJSON(w, data) writeSuccessResponseJSON(w, data)
} }
// ListBucketTargetsHandler - lists remote target(s) for a bucket or gets a target // ListRemoteTargetsHandler - lists remote target(s) for a bucket or gets a target
// for a particular ARN type // for a particular ARN type
func (a adminAPIHandlers) ListBucketTargetsHandler(w http.ResponseWriter, r *http.Request) { func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ListBucketTargets") ctx := newContext(r, w, "ListBucketTargets")
defer logger.AuditLog(w, r, "ListBucketTargets", mustGetClaimsFromToken(r)) defer logger.AuditLog(w, r, "ListBucketTargets", mustGetClaimsFromToken(r))
@ -225,28 +225,13 @@ func (a adminAPIHandlers) ListBucketTargetsHandler(w http.ResponseWriter, r *htt
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
return return
} }
if bucket != "" {
cfg, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket) if _, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket); err != nil {
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return return
} }
var (
targets []madmin.BucketTarget
tgt, ct madmin.BucketTarget
creds auth.Credentials
)
if cfg != nil && !cfg.Empty() {
for idx, t := range cfg.Targets {
if string(t.Type) == arnType || arnType == "" {
ct = cfg.Targets[idx]
// remove secretKey from creds
creds.AccessKey = ct.Credentials.AccessKey
tgt = madmin.BucketTarget{Endpoint: ct.Endpoint, Secure: ct.Secure, TargetBucket: ct.TargetBucket, Credentials: &creds, Arn: ct.Arn, Type: ct.Type}
targets = append(targets, tgt)
}
}
} }
targets := globalBucketTargetSys.ListTargets(ctx, bucket, arnType)
data, err := json.Marshal(targets) data, err := json.Marshal(targets)
if err != nil { if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@ -256,8 +241,8 @@ func (a adminAPIHandlers) ListBucketTargetsHandler(w http.ResponseWriter, r *htt
writeSuccessResponseJSON(w, data) writeSuccessResponseJSON(w, data)
} }
// RemoveBucketTargetHandler - removes a remote target for bucket with specified ARN // RemoveRemoteTargetHandler - removes a remote target for bucket with specified ARN
func (a adminAPIHandlers) RemoveBucketTargetHandler(w http.ResponseWriter, r *http.Request) { func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "RemoveBucketTarget") ctx := newContext(r, w, "RemoveBucketTarget")
defer logger.AuditLog(w, r, "RemoveBucketTarget", mustGetClaimsFromToken(r)) defer logger.AuditLog(w, r, "RemoveBucketTarget", mustGetClaimsFromToken(r))
@ -275,11 +260,18 @@ func (a adminAPIHandlers) RemoveBucketTargetHandler(w http.ResponseWriter, r *ht
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return return
} }
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, arn); err != nil { if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, arn); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return return
} }
targets, err := globalBucketTargetSys.ListTargets(ctx, bucket) targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return

View File

@ -183,14 +183,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
} }
// Bucket replication operations // Bucket replication operations
// GetBucketTargetHandler // GetBucketTargetHandler
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-bucket-targets").HandlerFunc( adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
httpTraceHdrs(adminAPI.ListBucketTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}") httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
// SetBucketTargetHandler // SetRemoteTargetHandler
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-target").HandlerFunc( adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
httpTraceHdrs(adminAPI.SetBucketTargetHandler)).Queries("bucket", "{bucket:.*}") httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
// SetBucketTargetHandler // SetRemoteTargetHandler
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-bucket-target").HandlerFunc( adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
httpTraceHdrs(adminAPI.RemoveBucketTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}") httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
} }
// -- Top APIs -- // -- Top APIs --

View File

@ -115,6 +115,7 @@ const (
ErrBucketRemoteArnInvalid ErrBucketRemoteArnInvalid
ErrBucketRemoteRemoveDisallowed ErrBucketRemoteRemoveDisallowed
ErrReplicationTargetNotVersionedError ErrReplicationTargetNotVersionedError
ErrReplicationSourceNotVersionedError
ErrReplicationNeedsVersioningError ErrReplicationNeedsVersioningError
ErrReplicationBucketNeedsVersioningError ErrReplicationBucketNeedsVersioningError
ErrBucketReplicationDisabledError ErrBucketReplicationDisabledError
@ -871,6 +872,11 @@ var errorCodes = errorCodeMap{
Description: "The replication target does not have versioning enabled", Description: "The replication target does not have versioning enabled",
HTTPStatusCode: http.StatusBadRequest, HTTPStatusCode: http.StatusBadRequest,
}, },
ErrReplicationSourceNotVersionedError: {
Code: "ReplicationSourceNotVersionedError",
Description: "The replication source does not have versioning enabled",
HTTPStatusCode: http.StatusBadRequest,
},
ErrReplicationNeedsVersioningError: { ErrReplicationNeedsVersioningError: {
Code: "InvalidRequest", Code: "InvalidRequest",
Description: "Versioning must be 'Enabled' on the bucket to apply a replication configuration", Description: "Versioning must be 'Enabled' on the bucket to apply a replication configuration",
@ -1929,6 +1935,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrBucketRemoteRemoveDisallowed apiErr = ErrBucketRemoteRemoveDisallowed
case BucketReplicationTargetNotVersioned: case BucketReplicationTargetNotVersioned:
apiErr = ErrReplicationTargetNotVersionedError apiErr = ErrReplicationTargetNotVersionedError
case BucketReplicationSourceNotVersioned:
apiErr = ErrReplicationSourceNotVersionedError
case BucketQuotaExceeded: case BucketQuotaExceeded:
apiErr = ErrAdminBucketQuotaExceeded apiErr = ErrAdminBucketQuotaExceeded
case *event.ErrInvalidEventName: case *event.ErrInvalidEventName:

View File

@ -49,7 +49,7 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
// validateReplicationDestination returns error if replication destination bucket missing or not configured // validateReplicationDestination returns error if replication destination bucket missing or not configured
// It also returns true if replication destination is same as this server. // It also returns true if replication destination is same as this server.
func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) { func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
clnt := globalBucketTargetSys.GetReplicationTargetClient(ctx, rCfg.ReplicationArn) clnt := globalBucketTargetSys.GetReplicationTargetClient(ctx, rCfg.RoleArn)
if clnt == nil { if clnt == nil {
return false, BucketRemoteTargetNotFound{Bucket: bucket} return false, BucketRemoteTargetNotFound{Bucket: bucket}
} }
@ -65,7 +65,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
} }
} }
// validate replication ARN against target endpoint // validate replication ARN against target endpoint
c, ok := globalBucketTargetSys.arnRemotesMap[rCfg.ReplicationArn] c, ok := globalBucketTargetSys.arnRemotesMap[rCfg.RoleArn]
if ok { if ok {
if c.EndpointURL().String() == clnt.EndpointURL().String() { if c.EndpointURL().String() == clnt.EndpointURL().String() {
sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort) sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
@ -159,7 +159,7 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
return return
} }
tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.ReplicationArn) tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.RoleArn)
if tgt == nil { if tgt == nil {
return return
} }

View File

@ -18,7 +18,6 @@ package cmd
import ( import (
"context" "context"
"fmt"
"net/http" "net/http"
"sync" "sync"
@ -36,16 +35,42 @@ type BucketTargetSys struct {
clientsCache map[string]*miniogo.Core clientsCache map[string]*miniogo.Core
} }
// ListTargets - gets list of bucket targets for this bucket. // ListTargets lists bucket targets across tenant or for individual bucket, and returns
func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) { // results filtered by arnType
if globalIsGateway { func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType string) (targets []madmin.BucketTarget) {
return nil, nil if bucket != "" {
if ts, err := sys.ListBucketTargets(ctx, bucket); err == nil {
for _, t := range ts.Targets {
if string(t.Type) == arnType || arnType == "" {
targets = append(targets, t.Clone())
} }
}
}
return targets
}
sys.RLock()
defer sys.RUnlock()
for _, tgts := range sys.targetsMap {
for _, t := range tgts {
if string(t.Type) == arnType || arnType == "" {
targets = append(targets, t.Clone())
}
}
}
return
}
// ListBucketTargets - gets list of bucket targets for this bucket.
func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
sys.RLock()
defer sys.RUnlock()
tgts, ok := sys.targetsMap[bucket] tgts, ok := sys.targetsMap[bucket]
if ok { if ok {
return &madmin.BucketTargets{Targets: tgts}, nil return &madmin.BucketTargets{Targets: tgts}, nil
} }
return nil, fmt.Errorf("No remote targets exist for bucket %s", bucket) return nil, BucketRemoteTargetNotFound{Bucket: bucket}
} }
// SetTarget - sets a new minio-go client target for this bucket. // SetTarget - sets a new minio-go client target for this bucket.
@ -60,8 +85,10 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
if err != nil { if err != nil {
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket} return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
} }
if tgt.Type == madmin.ReplicationService {
if tgt.Type == madmin.ReplicationArn { if !globalBucketVersioningSys.Enabled(bucket) {
return BucketReplicationSourceNotVersioned{Bucket: bucket}
}
vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket) vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
if err != nil || vcfg.Status != string(versioning.Enabled) { if err != nil || vcfg.Status != string(versioning.Enabled) {
if isErrBucketNotFound(err) { if isErrBucketNotFound(err) {
@ -112,10 +139,10 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
if err != nil { if err != nil {
return BucketRemoteArnInvalid{Bucket: bucket} return BucketRemoteArnInvalid{Bucket: bucket}
} }
if arn.Type == madmin.ReplicationArn { if arn.Type == madmin.ReplicationService {
// reject removal of remote target if replication configuration is present // reject removal of remote target if replication configuration is present
rcfg, err := getReplicationConfig(ctx, bucket) rcfg, err := getReplicationConfig(ctx, bucket)
if err == nil && rcfg.ReplicationArn == arnStr { if err == nil && rcfg.RoleArn == arnStr {
if _, ok := sys.arnRemotesMap[arnStr]; ok { if _, ok := sys.arnRemotesMap[arnStr]; ok {
return BucketRemoteRemoveDisallowed{Bucket: bucket} return BucketRemoteRemoveDisallowed{Bucket: bucket}
} }
@ -125,11 +152,17 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
sys.Lock() sys.Lock()
defer sys.Unlock() defer sys.Unlock()
targets := make([]madmin.BucketTarget, 0) targets := make([]madmin.BucketTarget, 0)
found := false
tgts := sys.targetsMap[bucket] tgts := sys.targetsMap[bucket]
for _, tgt := range tgts { for _, tgt := range tgts {
if tgt.Arn != arnStr { if tgt.Arn != arnStr {
targets = append(targets, tgt) targets = append(targets, tgt)
continue
} }
found = true
}
if !found {
return BucketRemoteTargetNotFound{Bucket: bucket}
} }
sys.targetsMap[bucket] = targets sys.targetsMap[bucket] = targets
delete(sys.arnRemotesMap, arnStr) delete(sys.arnRemotesMap, arnStr)
@ -168,6 +201,40 @@ func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objA
return nil return nil
} }
// UpdateTarget updates target to reflect metadata updates
func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTargets) {
if sys == nil {
return
}
sys.Lock()
defer sys.Unlock()
if cfg == nil || cfg.Empty() {
// remove target and arn association
if tgts, ok := sys.targetsMap[bucket]; ok {
for _, t := range tgts {
delete(sys.arnRemotesMap, t.Arn)
}
}
delete(sys.targetsMap, bucket)
return
}
if len(cfg.Targets) > 0 {
sys.targetsMap[bucket] = cfg.Targets
}
for _, tgt := range cfg.Targets {
tgtClient, err := sys.getRemoteTargetClient(&tgt)
if err != nil {
continue
}
sys.arnRemotesMap[tgt.Arn] = tgtClient
if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
}
}
sys.targetsMap[bucket] = cfg.Targets
}
// create minio-go clients for buckets having remote targets // create minio-go clients for buckets having remote targets
func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) { func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
for _, bucket := range buckets { for _, bucket := range buckets {
@ -229,7 +296,7 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
return tgt.Arn return tgt.Arn
} }
} }
if !madmin.ArnType(target.Type).IsValid() { if !madmin.ServiceType(target.Type).IsValid() {
return "" return ""
} }
arn := madmin.ARN{ arn := madmin.ARN{

View File

@ -411,6 +411,13 @@ func (e BucketReplicationTargetNotVersioned) Error() string {
return "Replication target does not have versioning enabled: " + e.Bucket return "Replication target does not have versioning enabled: " + e.Bucket
} }
// BucketReplicationSourceNotVersioned replication source does not have versioning enabled.
type BucketReplicationSourceNotVersioned GenericError
func (e BucketReplicationSourceNotVersioned) Error() string {
return "Replication source does not have versioning enabled: " + e.Bucket
}
/// Bucket related errors. /// Bucket related errors.
// BucketNameInvalid - bucketname provided is invalid. // BucketNameInvalid - bucketname provided is invalid.

View File

@ -610,6 +610,10 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
if meta.notificationConfig != nil { if meta.notificationConfig != nil {
globalNotificationSys.AddRulesMap(bucketName, meta.notificationConfig.ToRulesMap()) globalNotificationSys.AddRulesMap(bucketName, meta.notificationConfig.ToRulesMap())
} }
if meta.bucketTargetConfig != nil {
globalBucketTargetSys.UpdateTarget(bucketName, meta.bucketTargetConfig)
}
} }
// ReloadFormatHandler - Reload Format. // ReloadFormatHandler - Reload Format.

View File

@ -13,22 +13,22 @@ To replicate objects in a bucket to a destination bucket on a target site either
Create a replication target on the source cluster as shown below: Create a replication target on the source cluster as shown below:
``` ```
mc admin bucket remote set myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1 mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1
Replication ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket' Role ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket'
``` ```
Note that the admin needs *s3:GetReplicationConfigurationAction* permission on source cluster. The credential used at the destination requires *s3:ReplicateObject* permission. Once successfully created and authorized this generates a replication target ARN. The command below lists all the currently authorized replication targets: Note that the admin needs *s3:GetReplicationConfigurationAction* permission on source cluster. The credential used at the destination requires *s3:ReplicateObject* permission. Once successfully created and authorized this generates a replication target ARN. The command below lists all the currently authorized replication targets:
``` ```
mc admin bucket remote list myminio/srcbucket --service "replication" mc admin bucket remote ls myminio/srcbucket --service "replication"
Replication ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket' Role ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket'
``` ```
The replication configuration can now be added to the source bucket by applying the json file with replication configuration. The ReplicationArn is passed in as a json element in the configuration. The replication configuration can now be added to the source bucket by applying the json file with replication configuration. The Role ARN above is passed in as a json element in the configuration.
```json ```json
{ {
"ReplicationArn" :"arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket", "Role" :"arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket",
"Rules": [ "Rules": [
{ {
"Status": "Enabled", "Status": "Enabled",
@ -63,7 +63,7 @@ mc replicate add myminio/srcbucket/Tax --priority 1 --arn "arn:minio:replication
Replication configuration applied successfully to myminio/srcbucket. Replication configuration applied successfully to myminio/srcbucket.
``` ```
Apart from *ReplicationArn* , rest of the configuration follows [AWS S3 Spec](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). Any objects uploaded to the source bucket that meet replication criteria will now be automatically replicated by the MinIO server to the remote destination bucket. Replication can be disabled at any time by disabling specific rules in the configuration or deleting the replication configuration entirely. The replication configuration follows [AWS S3 Spec](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). Any objects uploaded to the source bucket that meet replication criteria will now be automatically replicated by the MinIO server to the remote destination bucket. Replication can be disabled at any time by disabling specific rules in the configuration or deleting the replication configuration entirely.
When an object is deleted from the source bucket, the replica will not be deleted as per S3 spec. When an object is deleted from the source bucket, the replica will not be deleted as per S3 spec.

View File

@ -50,7 +50,7 @@ var (
errReplicationNoRule = Errorf("Replication configuration should have at least one rule") errReplicationNoRule = Errorf("Replication configuration should have at least one rule")
errReplicationUniquePriority = Errorf("Replication configuration has duplicate priority") errReplicationUniquePriority = Errorf("Replication configuration has duplicate priority")
errReplicationDestinationMismatch = Errorf("The destination bucket must be same for all rules") errReplicationDestinationMismatch = Errorf("The destination bucket must be same for all rules")
errReplicationArnMissing = Errorf("Replication Arn missing") errRoleArnMissing = Errorf("Missing required parameter `Role` in ReplicationConfiguration")
) )
// Config - replication configuration specified in // Config - replication configuration specified in
@ -58,8 +58,8 @@ var (
type Config struct { type Config struct {
XMLName xml.Name `xml:"ReplicationConfiguration" json:"-"` XMLName xml.Name `xml:"ReplicationConfiguration" json:"-"`
Rules []Rule `xml:"Rule" json:"Rules"` Rules []Rule `xml:"Rule" json:"Rules"`
// ReplicationArn is a MinIO only extension and optional for AWS // RoleArn is being reused for MinIO replication ARN
ReplicationArn string `xml:"ReplicationArn,omitempty" json:"ReplicationArn,omitempty"` RoleArn string `xml:"Role" json:"Role"`
} }
// Maximum 2MiB size per replication config. // Maximum 2MiB size per replication config.
@ -84,8 +84,8 @@ func (c Config) Validate(bucket string, sameTarget bool) error {
if len(c.Rules) == 0 { if len(c.Rules) == 0 {
return errReplicationNoRule return errReplicationNoRule
} }
if c.ReplicationArn == "" { if c.RoleArn == "" {
return errReplicationArnMissing return errRoleArnMissing
} }
// Validate all the rules in the replication config // Validate all the rules in the replication config
targetMap := make(map[string]struct{}) targetMap := make(map[string]struct{})

View File

@ -29,22 +29,22 @@ import (
"github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/auth"
) )
// ArnType represents bucket ARN type // ServiceType represents service type
type ArnType string type ServiceType string
const ( const (
// ReplicationArn specifies a ARN type of replication // ReplicationService specifies replication service
ReplicationArn ArnType = "replication" ReplicationService ServiceType = "replication"
) )
// IsValid returns true if ARN type is replication // IsValid returns true if ARN type represents replication
func (t ArnType) IsValid() bool { func (t ServiceType) IsValid() bool {
return t == ReplicationArn return t == ReplicationService
} }
// ARN is a struct to define arn. // ARN is a struct to define arn.
type ARN struct { type ARN struct {
Type ArnType Type ServiceType
ID string ID string
Region string Region string
Bucket string Bucket string
@ -75,7 +75,7 @@ func ParseARN(s string) (*ARN, error) {
} }
return &ARN{ return &ARN{
Type: ArnType(tokens[2]), Type: ServiceType(tokens[2]),
Region: tokens[3], Region: tokens[3],
ID: tokens[4], ID: tokens[4],
Bucket: tokens[5], Bucket: tokens[5],
@ -84,6 +84,7 @@ func ParseARN(s string) (*ARN, error) {
// BucketTarget represents the target bucket and site association. // BucketTarget represents the target bucket and site association.
type BucketTarget struct { type BucketTarget struct {
SourceBucket string `json:"sourcebucket"`
Endpoint string `json:"endpoint"` Endpoint string `json:"endpoint"`
Credentials *auth.Credentials `json:"credentials"` Credentials *auth.Credentials `json:"credentials"`
TargetBucket string `json:"targetbucket"` TargetBucket string `json:"targetbucket"`
@ -91,10 +92,26 @@ type BucketTarget struct {
Path string `json:"path,omitempty"` Path string `json:"path,omitempty"`
API string `json:"api,omitempty"` API string `json:"api,omitempty"`
Arn string `json:"arn,omitempty"` Arn string `json:"arn,omitempty"`
Type ArnType `json:"type"` Type ServiceType `json:"type"`
Region string `json:"omitempty"` Region string `json:"omitempty"`
} }
// Clone returns shallow clone of BucketTarget without secret key in credentials
func (t *BucketTarget) Clone() BucketTarget {
return BucketTarget{
SourceBucket: t.SourceBucket,
Endpoint: t.Endpoint,
TargetBucket: t.TargetBucket,
Credentials: &auth.Credentials{AccessKey: t.Credentials.AccessKey},
Secure: t.Secure,
Path: t.Path,
API: t.Path,
Arn: t.Arn,
Type: t.Type,
Region: t.Region,
}
}
// URL returns target url // URL returns target url
func (t BucketTarget) URL() string { func (t BucketTarget) URL() string {
scheme := "http" scheme := "http"
@ -132,18 +149,18 @@ func (t BucketTargets) Empty() bool {
return empty return empty
} }
// ListBucketTargets - gets target(s) for this bucket // ListRemoteTargets - gets target(s) for this bucket
func (adm *AdminClient) ListBucketTargets(ctx context.Context, bucket, arnType string) (targets []BucketTarget, err error) { func (adm *AdminClient) ListRemoteTargets(ctx context.Context, bucket, arnType string) (targets []BucketTarget, err error) {
queryValues := url.Values{} queryValues := url.Values{}
queryValues.Set("bucket", bucket) queryValues.Set("bucket", bucket)
queryValues.Set("type", arnType) queryValues.Set("type", arnType)
reqData := requestData{ reqData := requestData{
relPath: adminAPIPrefix + "/list-bucket-targets", relPath: adminAPIPrefix + "/list-remote-targets",
queryValues: queryValues, queryValues: queryValues,
} }
// Execute GET on /minio/admin/v3/list-bucket-targets // Execute GET on /minio/admin/v3/list-remote-targets
resp, err := adm.executeMethod(ctx, http.MethodGet, reqData) resp, err := adm.executeMethod(ctx, http.MethodGet, reqData)
defer closeResponse(resp) defer closeResponse(resp)
@ -165,8 +182,8 @@ func (adm *AdminClient) ListBucketTargets(ctx context.Context, bucket, arnType s
return targets, nil return targets, nil
} }
// SetBucketTarget sets up a remote target for this bucket // SetRemoteTarget sets up a remote target for this bucket
func (adm *AdminClient) SetBucketTarget(ctx context.Context, bucket string, target *BucketTarget) (string, error) { func (adm *AdminClient) SetRemoteTarget(ctx context.Context, bucket string, target *BucketTarget) (string, error) {
data, err := json.Marshal(target) data, err := json.Marshal(target)
if err != nil { if err != nil {
return "", err return "", err
@ -179,12 +196,12 @@ func (adm *AdminClient) SetBucketTarget(ctx context.Context, bucket string, targ
queryValues.Set("bucket", bucket) queryValues.Set("bucket", bucket)
reqData := requestData{ reqData := requestData{
relPath: adminAPIPrefix + "/set-bucket-target", relPath: adminAPIPrefix + "/set-remote-target",
queryValues: queryValues, queryValues: queryValues,
content: encData, content: encData,
} }
// Execute PUT on /minio/admin/v3/set-bucket-target to set a target for this bucket of specific arn type. // Execute PUT on /minio/admin/v3/set-remote-target to set a target for this bucket of specific arn type.
resp, err := adm.executeMethod(ctx, http.MethodPut, reqData) resp, err := adm.executeMethod(ctx, http.MethodPut, reqData)
defer closeResponse(resp) defer closeResponse(resp)
@ -206,18 +223,18 @@ func (adm *AdminClient) SetBucketTarget(ctx context.Context, bucket string, targ
return arn, nil return arn, nil
} }
// RemoveBucketTarget removes a remote target associated with particular ARN for this bucket // RemoveRemoteTarget removes a remote target associated with particular ARN for this bucket
func (adm *AdminClient) RemoveBucketTarget(ctx context.Context, bucket, arn string) error { func (adm *AdminClient) RemoveRemoteTarget(ctx context.Context, bucket, arn string) error {
queryValues := url.Values{} queryValues := url.Values{}
queryValues.Set("bucket", bucket) queryValues.Set("bucket", bucket)
queryValues.Set("arn", arn) queryValues.Set("arn", arn)
reqData := requestData{ reqData := requestData{
relPath: adminAPIPrefix + "/remove-bucket-target", relPath: adminAPIPrefix + "/remove-remote-target",
queryValues: queryValues, queryValues: queryValues,
} }
// Execute PUT on /minio/admin/v3/remove-bucket-target to remove a target for this bucket // Execute PUT on /minio/admin/v3/remove-remote-target to remove a target for this bucket
// with specific ARN // with specific ARN
resp, err := adm.executeMethod(ctx, http.MethodDelete, reqData) resp, err := adm.executeMethod(ctx, http.MethodDelete, reqData)
defer closeResponse(resp) defer closeResponse(resp)