Add support for existing object replication. (#12109)

Also adding an API to allow resyncing replication when
existing object replication is enabled and the remote target
is entirely lost. With the `mc replicate reset` command, the
objects that are eligible for replication as per the replication
config will be resynced to target if existing object replication
is enabled on the rule.

internal/bucket/replication/replication.go

@@ -131,23 +131,31 @@ type Type int
 
 // Types of replication
 const (
-	ObjectReplicationType Type = 1 + iota
+	UnsetReplicationType Type = 0 + iota
+	ObjectReplicationType
 	DeleteReplicationType
 	MetadataReplicationType
 	HealReplicationType
+	ExistingObjectReplicationType
 )
 
+// Valid returns true if replication type is set
+func (t Type) Valid() bool {
+	return t > 0
+}
+
 // ObjectOpts provides information to deduce whether replication
 // can be triggered on the resultant object.
 type ObjectOpts struct {
-	Name         string
-	UserTags     string
-	VersionID    string
-	IsLatest     bool
-	DeleteMarker bool
-	SSEC         bool
-	OpType       Type
-	Replica      bool
+	Name           string
+	UserTags       string
+	VersionID      string
+	IsLatest       bool
+	DeleteMarker   bool
+	SSEC           bool
+	OpType         Type
+	Replica        bool
+	ExistingObject bool
 }
 
 // FilterActionableRules returns the rules actions that need to be executed
@@ -191,6 +199,9 @@ func (c Config) Replicate(obj ObjectOpts) bool {
 		if rule.Status == Disabled {
 			continue
 		}
+		if obj.ExistingObject && rule.ExistingObjectReplication.Status == Disabled {
+			return false
+		}
 		if obj.OpType == DeleteReplicationType {
 			switch {
 			case obj.VersionID != "":

internal/bucket/replication/rule.go

@@ -90,6 +90,43 @@ func (d *DeleteReplication) UnmarshalXML(dec *xml.Decoder, start xml.StartElemen
 	return nil
 }
 
+// ExistingObjectReplication - whether existing object replication is enabled
+type ExistingObjectReplication struct {
+	Status Status `xml:"Status"` // should be set to "Disabled" by default
+}
+
+// IsEmpty returns true if ExistingObjectReplication is not set
+func (e ExistingObjectReplication) IsEmpty() bool {
+	return len(e.Status) == 0
+}
+
+// Validate validates whether the status is disabled.
+func (e ExistingObjectReplication) Validate() error {
+	if e.IsEmpty() {
+		return nil
+	}
+	if e.Status != Disabled && e.Status != Enabled {
+		return errInvalidExistingObjectReplicationStatus
+	}
+	return nil
+}
+
+// UnmarshalXML - decodes XML data. Default to Disabled unless specified
+func (e *ExistingObjectReplication) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) {
+	// Make subtype to avoid recursive UnmarshalXML().
+	type existingObjectReplication ExistingObjectReplication
+	erep := existingObjectReplication{}
+
+	if err := dec.DecodeElement(&erep, &start); err != nil {
+		return err
+	}
+	if len(erep.Status) == 0 {
+		erep.Status = Disabled
+	}
+	e.Status = erep.Status
+	return nil
+}
+
 // Rule - a rule for replication configuration.
 type Rule struct {
 	XMLName                 xml.Name                `xml:"Rule" json:"Rule"`
@@ -98,22 +135,24 @@ type Rule struct {
 	Priority                int                     `xml:"Priority" json:"Priority"`
 	DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication" json:"DeleteMarkerReplication"`
 	// MinIO extension to replicate versioned deletes
-	DeleteReplication       DeleteReplication       `xml:"DeleteReplication" json:"DeleteReplication"`
-	Destination             Destination             `xml:"Destination" json:"Destination"`
-	SourceSelectionCriteria SourceSelectionCriteria `xml:"SourceSelectionCriteria" json:"SourceSelectionCriteria"`
-	Filter                  Filter                  `xml:"Filter" json:"Filter"`
+	DeleteReplication         DeleteReplication         `xml:"DeleteReplication" json:"DeleteReplication"`
+	Destination               Destination               `xml:"Destination" json:"Destination"`
+	SourceSelectionCriteria   SourceSelectionCriteria   `xml:"SourceSelectionCriteria" json:"SourceSelectionCriteria"`
+	Filter                    Filter                    `xml:"Filter" json:"Filter"`
+	ExistingObjectReplication ExistingObjectReplication `xml:"ExistingObjectReplication,omitempty" json:"ExistingObjectReplication,omitempty"`
 }
 
 var (
-	errInvalidRuleID                        = Errorf("ID must be less than 255 characters")
-	errEmptyRuleStatus                      = Errorf("Status should not be empty")
-	errInvalidRuleStatus                    = Errorf("Status must be set to either Enabled or Disabled")
-	errDeleteMarkerReplicationMissing       = Errorf("DeleteMarkerReplication must be specified")
-	errPriorityMissing                      = Errorf("Priority must be specified")
-	errInvalidDeleteMarkerReplicationStatus = Errorf("Delete marker replication status is invalid")
-	errDestinationSourceIdentical           = Errorf("Destination bucket cannot be the same as the source bucket.")
-	errDeleteReplicationMissing             = Errorf("Delete replication must be specified")
-	errInvalidDeleteReplicationStatus       = Errorf("Delete replication is either enable|disable")
+	errInvalidRuleID                          = Errorf("ID must be less than 255 characters")
+	errEmptyRuleStatus                        = Errorf("Status should not be empty")
+	errInvalidRuleStatus                      = Errorf("Status must be set to either Enabled or Disabled")
+	errDeleteMarkerReplicationMissing         = Errorf("DeleteMarkerReplication must be specified")
+	errPriorityMissing                        = Errorf("Priority must be specified")
+	errInvalidDeleteMarkerReplicationStatus   = Errorf("Delete marker replication status is invalid")
+	errDestinationSourceIdentical             = Errorf("Destination bucket cannot be the same as the source bucket.")
+	errDeleteReplicationMissing               = Errorf("Delete replication must be specified")
+	errInvalidDeleteReplicationStatus         = Errorf("Delete replication is either enable|disable")
+	errInvalidExistingObjectReplicationStatus = Errorf("Existing object replication status is invalid")
 )
 
 // validateID - checks if ID is valid or not.
@@ -200,7 +239,7 @@ func (r Rule) Validate(bucket string, sameTarget bool) error {
 	if r.Destination.Bucket == bucket && sameTarget {
 		return errDestinationSourceIdentical
 	}
-	return nil
+	return r.ExistingObjectReplication.Validate()
 }
 
 // MetadataReplicate  returns true if object is not a replica or in the case of replicas,

internal/http/headers.go

@@ -170,6 +170,9 @@ const (
 	MinIOSourceProxyRequest = "X-Minio-Source-Proxy-Request"
 	// Header indicates that this request is a replication request to create a REPLICA
 	MinIOSourceReplicationRequest = "X-Minio-Source-Replication-Request"
+	// Header indicates replication reset status.
+	MinIOReplicationResetStatus = "X-Minio-Replication-Reset-Status"
+
 	// predicted date/time of transition
 	MinIOTransition = "X-Minio-Transition"
 )