Add support for multi site replication (#12880)

This commit is contained in:
Poorna Krishnamoorthy
2021-09-18 16:31:35 -04:00
committed by GitHub
parent 0b8c5a6872
commit c4373ef290
52 changed files with 6492 additions and 1230 deletions

View File

@@ -0,0 +1,47 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package replication
//go:generate msgp -file=$GOFILE
// StatusType of Replication for x-amz-replication-status header
type StatusType string
const (
// Pending - replication is pending.
Pending StatusType = "PENDING"
// Completed - replication completed ok.
Completed StatusType = "COMPLETED"
// Failed - replication failed.
Failed StatusType = "FAILED"
// Replica - this is a replica.
Replica StatusType = "REPLICA"
)
// String returns string representation of status
func (s StatusType) String() string {
return string(s)
}
// Empty returns true if this status is not set
func (s StatusType) Empty() bool {
return string(s) == ""
}

View File

@@ -0,0 +1,59 @@
package replication
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
import (
"github.com/tinylib/msgp/msgp"
)
// DecodeMsg implements msgp.Decodable
func (z *StatusType) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zb0001 string
zb0001, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = StatusType(zb0001)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z StatusType) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteString(string(z))
if err != nil {
err = msgp.WrapError(err)
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z StatusType) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendString(o, string(z))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *StatusType) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zb0001 string
zb0001, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = StatusType(zb0001)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z StatusType) Msgsize() (s int) {
s = msgp.StringPrefixSize + len(string(z))
return
}

View File

@@ -0,0 +1,3 @@
package replication
// Code generated by github.com/tinylib/msgp DO NOT EDIT.

View File

@@ -28,11 +28,15 @@ import (
// DestinationARNPrefix - destination ARN prefix as per AWS S3 specification.
const DestinationARNPrefix = "arn:aws:s3:::"
// DestinationARNMinIOPrefix - destination ARN prefix for MinIO.
const DestinationARNMinIOPrefix = "arn:minio:replication:"
// Destination - destination in ReplicationConfiguration.
type Destination struct {
XMLName xml.Name `xml:"Destination" json:"Destination"`
Bucket string `xml:"Bucket" json:"Bucket"`
StorageClass string `xml:"StorageClass" json:"StorageClass"`
ARN string
//EncryptionConfiguration TODO: not needed for MinIO
}
@@ -49,7 +53,20 @@ func (d Destination) IsValid() bool {
}
func (d Destination) String() string {
return DestinationARNPrefix + d.Bucket
return d.ARN
}
//LegacyArn returns true if arn format has prefix "arn:aws:s3:::" which was used
// prior to multi-destination
func (d Destination) LegacyArn() bool {
return strings.HasPrefix(d.ARN, DestinationARNPrefix)
}
//TargetArn returns true if arn format has prefix "arn:minio:replication:::" used
// for multi-destination targets
func (d Destination) TargetArn() bool {
return strings.HasPrefix(d.ARN, DestinationARNMinIOPrefix)
}
// MarshalXML - encodes to XML data.
@@ -107,7 +124,7 @@ func (d Destination) Validate(bucketName string) error {
// parseDestination - parses string to Destination.
func parseDestination(s string) (Destination, error) {
if !strings.HasPrefix(s, DestinationARNPrefix) {
if !strings.HasPrefix(s, DestinationARNPrefix) && !strings.HasPrefix(s, DestinationARNMinIOPrefix) {
return Destination{}, Errorf("invalid destination '%s'", s)
}
@@ -115,5 +132,6 @@ func parseDestination(s string) (Destination, error) {
return Destination{
Bucket: bucketName,
ARN: s,
}, nil
}

View File

@@ -25,40 +25,14 @@ import (
"strings"
)
// StatusType of Replication for x-amz-replication-status header
type StatusType string
const (
// Pending - replication is pending.
Pending StatusType = "PENDING"
// Completed - replication completed ok.
Completed StatusType = "COMPLETED"
// Failed - replication failed.
Failed StatusType = "FAILED"
// Replica - this is a replica.
Replica StatusType = "REPLICA"
)
// String returns string representation of status
func (s StatusType) String() string {
return string(s)
}
// Empty returns true if this status is not set
func (s StatusType) Empty() bool {
return string(s) == ""
}
var (
errReplicationTooManyRules = Errorf("Replication configuration allows a maximum of 1000 rules")
errReplicationNoRule = Errorf("Replication configuration should have at least one rule")
errReplicationUniquePriority = Errorf("Replication configuration has duplicate priority")
errReplicationDestinationMismatch = Errorf("The destination bucket must be same for all rules")
errRoleArnMissing = Errorf("Missing required parameter `Role` in ReplicationConfiguration")
errInvalidSourceSelectionCriteria = Errorf("Invalid ReplicaModification status")
errReplicationTooManyRules = Errorf("Replication configuration allows a maximum of 1000 rules")
errReplicationNoRule = Errorf("Replication configuration should have at least one rule")
errReplicationUniquePriority = Errorf("Replication configuration has duplicate priority")
errRoleArnMissingLegacy = Errorf("Missing required parameter `Role` in ReplicationConfiguration")
errDestinationArnMissing = Errorf("Missing required parameter `Destination` in Replication rule")
errInvalidSourceSelectionCriteria = Errorf("Invalid ReplicaModification status")
errRoleArnPresentForMultipleTargets = Errorf("`Role` should be empty in ReplicationConfiguration for multiple targets")
)
// Config - replication configuration specified in
@@ -102,18 +76,14 @@ func (c Config) Validate(bucket string, sameTarget bool) error {
if len(c.Rules) == 0 {
return errReplicationNoRule
}
if c.RoleArn == "" {
return errRoleArnMissing
}
// Validate all the rules in the replication config
targetMap := make(map[string]struct{})
priorityMap := make(map[string]struct{})
var legacyArn bool
for _, r := range c.Rules {
if len(targetMap) == 0 {
targetMap[r.Destination.Bucket] = struct{}{}
}
if _, ok := targetMap[r.Destination.Bucket]; !ok {
return errReplicationDestinationMismatch
targetMap[r.Destination.Bucket] = struct{}{}
}
if err := r.Validate(bucket, sameTarget); err != nil {
return err
@@ -122,6 +92,22 @@ func (c Config) Validate(bucket string, sameTarget bool) error {
return errReplicationUniquePriority
}
priorityMap[strconv.Itoa(r.Priority)] = struct{}{}
if r.Destination.LegacyArn() {
legacyArn = true
}
if c.RoleArn == "" && !r.Destination.TargetArn() {
return errDestinationArnMissing
}
}
// disallow combining old replication configuration which used RoleArn as target ARN with multiple
// destination replication
if c.RoleArn != "" && len(targetMap) > 1 {
return errRoleArnPresentForMultipleTargets
}
// validate RoleArn if destination used legacy ARN format.
if c.RoleArn == "" && legacyArn {
return errRoleArnMissingLegacy
}
return nil
}
@@ -137,6 +123,7 @@ const (
MetadataReplicationType
HealReplicationType
ExistingObjectReplicationType
ResyncReplicationType
)
// Valid returns true if replication type is set
@@ -150,18 +137,18 @@ type ObjectOpts struct {
Name string
UserTags string
VersionID string
IsLatest bool
DeleteMarker bool
SSEC bool
OpType Type
Replica bool
ExistingObject bool
TargetArn string
}
// FilterActionableRules returns the rules actions that need to be executed
// after evaluating prefix/tag filtering
func (c Config) FilterActionableRules(obj ObjectOpts) []Rule {
if obj.Name == "" {
if obj.Name == "" && obj.OpType != ResyncReplicationType {
return nil
}
var rules []Rule
@@ -169,6 +156,18 @@ func (c Config) FilterActionableRules(obj ObjectOpts) []Rule {
if rule.Status == Disabled {
continue
}
if obj.TargetArn != "" && rule.Destination.ARN != obj.TargetArn && c.RoleArn != obj.TargetArn {
continue
}
// Ignore other object level and prefix filters for resyncing target
if obj.OpType == ResyncReplicationType {
rules = append(rules, rule)
continue
}
if obj.ExistingObject && rule.ExistingObjectReplication.Status == Disabled {
continue
}
if !strings.HasPrefix(obj.Name, rule.Prefix()) {
continue
}
@@ -177,8 +176,9 @@ func (c Config) FilterActionableRules(obj ObjectOpts) []Rule {
}
}
sort.Slice(rules[:], func(i, j int) bool {
return rules[i].Priority > rules[j].Priority
return rules[i].Priority > rules[j].Priority && rules[i].Destination.String() == rules[j].Destination.String()
})
return rules
}
@@ -205,7 +205,7 @@ func (c Config) Replicate(obj ObjectOpts) bool {
if obj.OpType == DeleteReplicationType {
switch {
case obj.VersionID != "":
// // check MinIO extension for versioned deletes
// check MinIO extension for versioned deletes
return rule.DeleteReplication.Status == Enabled
default:
return rule.DeleteMarkerReplication.Status == Enabled
@@ -243,3 +243,27 @@ func (c Config) HasActiveRules(prefix string, recursive bool) bool {
}
return false
}
// FilterTargetArns returns a slice of distinct target arns in the config
func (c Config) FilterTargetArns(obj ObjectOpts) []string {
var arns []string
tgtsMap := make(map[string]struct{})
rules := c.FilterActionableRules(obj)
for _, rule := range rules {
if rule.Status == Disabled {
continue
}
if c.RoleArn != "" {
arns = append(arns, c.RoleArn) // use legacy RoleArn if present
return arns
}
if _, ok := tgtsMap[rule.Destination.ARN]; !ok {
tgtsMap[rule.Destination.ARN] = struct{}{}
}
}
for k := range tgtsMap {
arns = append(arns, k)
}
return arns
}

View File

@@ -52,20 +52,20 @@ func TestParseAndValidateReplicationConfig(t *testing.T) {
expectedParsingErr: nil,
expectedValidationErr: nil,
},
//4 missing role in config
//4 missing role in config and destination ARN is in legacy format
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>key-prefix</Prefix><Destination><Bucket>arn:aws:s3:::destinationbucket</Bucket></Destination></Rule></ReplicationConfiguration>`,
// destination bucket in config different from bucket specified
destBucket: "destinationbucket",
sameTarget: false,
expectedParsingErr: nil,
expectedValidationErr: errRoleArnMissing,
expectedValidationErr: errDestinationArnMissing,
},
//5 replication destination in different rules not identical
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::AcctID:role/role-name</Role><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>key-prefix</Prefix><Destination><Bucket>arn:aws:s3:::destinationbucket</Bucket></Destination></Rule><Rule><Status>Enabled</Status><Priority>3</Priority><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>key-prefix</Prefix><Destination><Bucket>arn:aws:s3:::destinationbucket2</Bucket></Destination></Rule></ReplicationConfiguration>`,
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role></Role><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>key-prefix</Prefix><Destination><Bucket>arn:minio:replication:::destinationbucket</Bucket></Destination></Rule><Rule><Status>Enabled</Status><Priority>3</Priority><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>key-prefix</Prefix><Destination><Bucket>arn:minio:replication:::destinationbucket2</Bucket></Destination></Rule></ReplicationConfiguration>`,
destBucket: "destinationbucket",
sameTarget: false,
expectedParsingErr: nil,
expectedValidationErr: errReplicationDestinationMismatch,
expectedValidationErr: nil,
},
//6 missing rule status in replication config
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::AcctID:role/role-name</Role><Rule><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>key-prefix</Prefix><Destination><Bucket>arn:aws:s3:::destinationbucket</Bucket></Destination></Rule></ReplicationConfiguration>`,
@@ -116,6 +116,22 @@ func TestParseAndValidateReplicationConfig(t *testing.T) {
expectedParsingErr: fmt.Errorf("invalid destination '%v'", "destinationbucket2"),
expectedValidationErr: nil,
},
//13 missing role in config and destination ARN has target ARN
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>key-prefix</Prefix><Destination><Bucket>arn:minio:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket</Bucket></Destination></Rule></ReplicationConfiguration>`,
// destination bucket in config different from bucket specified
destBucket: "destinationbucket",
sameTarget: false,
expectedParsingErr: nil,
expectedValidationErr: nil,
},
//14 role absent in config and destination ARN has target ARN in invalid format
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>key-prefix</Prefix><Destination><Bucket>arn:xx:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket</Bucket></Destination></Rule></ReplicationConfiguration>`,
// destination bucket in config different from bucket specified
destBucket: "destinationbucket",
sameTarget: false,
expectedParsingErr: fmt.Errorf("invalid destination '%v'", "arn:xx:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket"),
expectedValidationErr: nil,
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
@@ -333,3 +349,52 @@ func TestHasActiveRules(t *testing.T) {
}
}
func TestFilterActionableRules(t *testing.T) {
testCases := []struct {
inputConfig string
prefix string
ExpectedRules []Rule
}{
// case 1 - only one rule
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::AcctID:role/role-name</Role><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>prefix</Prefix><Priority>1</Priority><Destination><Bucket>arn:minio:replication:xxx::destinationbucket</Bucket></Destination></Rule></ReplicationConfiguration>`,
prefix: "prefix",
ExpectedRules: []Rule{{Status: Enabled, Priority: 1, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}}},
},
// case 2 - multiple rules for same target, overlapping rules with different priority
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::AcctID:role/role-name</Role><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>prefix</Prefix><Priority>3</Priority><Destination><Bucket>arn:minio:replication:xxx::destinationbucket</Bucket></Destination></Rule><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>prefix</Prefix><Priority>1</Priority><Destination><Bucket>arn:minio:replication:xxx::destinationbucket</Bucket></Destination></Rule></ReplicationConfiguration>`,
prefix: "prefix",
ExpectedRules: []Rule{
{Status: Enabled, Priority: 3, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}},
{Status: Enabled, Priority: 1, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}},
},
},
// case 3 - multiple rules for different target, overlapping rules on a target
{inputConfig: `<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::AcctID:role/role-name</Role><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>prefix</Prefix><Priority>2</Priority><Destination><Bucket>arn:minio:replication:xxx::destinationbucket2</Bucket></Destination></Rule><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>prefix</Prefix><Priority>4</Priority><Destination><Bucket>arn:minio:replication:xxx::destinationbucket2</Bucket></Destination></Rule><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>prefix</Prefix><Priority>3</Priority><Destination><Bucket>arn:minio:replication:xxx::destinationbucket</Bucket></Destination></Rule><Rule><Status>Enabled</Status><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Disabled</Status></DeleteReplication><Prefix>prefix</Prefix><Priority>1</Priority><Destination><Bucket>arn:minio:replication:xxx::destinationbucket</Bucket></Destination></Rule></ReplicationConfiguration>`,
prefix: "prefix",
ExpectedRules: []Rule{
{Status: Enabled, Priority: 4, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket2", ARN: "arn:minio:replication:xxx::destinationbucket2"}},
{Status: Enabled, Priority: 2, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket2", ARN: "arn:minio:replication:xxx::destinationbucket2"}},
{Status: Enabled, Priority: 3, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}},
{Status: Enabled, Priority: 1, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}},
},
},
}
for _, tc := range testCases {
tc := tc
cfg, err := ParseConfig(bytes.NewReader([]byte(tc.inputConfig)))
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
got := cfg.FilterActionableRules(ObjectOpts{Name: tc.prefix})
if len(got) != len(tc.ExpectedRules) {
t.Fatalf("Expected matching number of actionable rules: `%v`, got: `%v`", tc.ExpectedRules, got)
}
for i := range got {
if got[i].Destination.ARN != tc.ExpectedRules[i].Destination.ARN || got[i].Priority != tc.ExpectedRules[i].Priority {
t.Fatalf("Expected order of filtered rules to be identical: `%v`, got: `%v`", tc.ExpectedRules, got)
}
}
}
}