Add support for existing object replication. (#12109)

Also adding an API to allow resyncing replication when
existing object replication is enabled and the remote target
is entirely lost. With the `mc replicate reset` command, the
objects that are eligible for replication as per the replication
config will be resynced to target if existing object replication
is enabled on the rule.

cmd/api-errors.go

@@ -125,6 +125,7 @@ const (
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
+	ErrReplicationNoMatchingRuleError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
 	ErrNoSuchUpload
@@ -859,6 +860,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Remote service connection error - please check remote service credentials and target bucket",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrReplicationNoMatchingRuleError: {
+		Code:           "XMinioReplicationNoMatchingRule",
+		Description:    "No matching replication rule found for this object prefix",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",

cmd/api-router.go

@@ -394,6 +394,9 @@ func registerAPIRouter(router *mux.Router) {
 		// PutBucketNotification
 		router.Methods(http.MethodPut).HandlerFunc(
 			collectAPIStats("putbucketnotification", maxClients(httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
+		// ResetBucketReplicationState - MinIO extension API
+		router.Methods(http.MethodPut).HandlerFunc(
+			collectAPIStats("resetbucketreplicationstate", maxClients(httpTraceAll(api.ResetBucketReplicationStateHandler)))).Queries("replication-reset", "")
 		// PutBucket
 		router.Methods(http.MethodPut).HandlerFunc(
 			collectAPIStats("putbucket", maxClients(httpTraceAll(api.PutBucketHandler))))

cmd/bucket-handlers.go

@@ -33,6 +33,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
@@ -611,7 +612,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 		if replicateDeletes {
 			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
-				dv := DeletedObjectVersionInfo{
+				dv := DeletedObjectReplicationInfo{
 					DeletedObject: dobj,
 					Bucket:        bucket,
 				}
@@ -1686,3 +1687,90 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	}
 	w.(http.Flusher).Flush()
 }
+
+// ResetBucketReplicationStateHandler - starts a replication reset for all objects in a bucket which
+// qualify for replication and re-sync the object(s) to target, provided ExistingObjectReplication is
+// enabled for the qualifying rule. This API is a MinIO only extension provided for situations where
+// remote target is entirely lost,and previously replicated objects need to be re-synced.
+func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ResetBucketReplicationState")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	durationStr := r.URL.Query().Get("older-than")
+	var (
+		days time.Duration
+		err  error
+	)
+	if durationStr != "" {
+		days, err = time.ParseDuration(durationStr)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, InvalidArgument{
+				Bucket: bucket,
+				Err:    fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
+			}), r.URL, guessIsBrowserReq(r))
+		}
+	}
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	config, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	if !config.HasActiveRules("", true) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoMatchingRuleError), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, config.RoleArn)
+	target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24))
+	target.ResetID = mustGetUUID()
+	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
+		switch err.(type) {
+		case BucketRemoteConnectionErr:
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
+		default:
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		}
+		return
+	}
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	tgtBytes, err := json.Marshal(&targets)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	data, err := json.Marshal(target.ResetID)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}

cmd/bucket-replication.go

@@ -91,32 +91,70 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 	return false, BucketRemoteTargetNotFound{Bucket: bucket}
 }
 
-func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) (replicate bool, sync bool) {
+type mustReplicateOptions struct {
+	meta   map[string]string
+	status replication.StatusType
+	opType replication.Type
+}
+
+func (o mustReplicateOptions) ReplicationStatus() (s replication.StatusType) {
+	if rs, ok := o.meta[xhttp.AmzBucketReplicationStatus]; ok {
+		return replication.StatusType(rs)
+	}
+	return s
+}
+func (o mustReplicateOptions) isExistingObjectReplication() bool {
+	return o.opType == replication.ExistingObjectReplicationType
+}
+
+func (o mustReplicateOptions) isMetadataReplication() bool {
+	return o.opType == replication.MetadataReplicationType
+}
+func getMustReplicateOptions(o ObjectInfo, op replication.Type) mustReplicateOptions {
+	if !op.Valid() {
+		op = replication.ObjectReplicationType
+		if o.metadataOnly {
+			op = replication.MetadataReplicationType
+		}
+	}
+	meta := cloneMSS(o.UserDefined)
+	if o.UserTags != "" {
+		meta[xhttp.AmzObjectTagging] = o.UserTags
+	}
+	return mustReplicateOptions{
+		meta:   meta,
+		status: o.ReplicationStatus,
+		opType: op,
+	}
+}
+func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus replication.StatusType, permErr APIErrorCode) (replicate bool, sync bool) {
 	if permErr != ErrNone {
 		return
 	}
-	return mustReplicater(ctx, bucket, object, meta, replStatus, false)
+	return mustReplicater(ctx, bucket, object, mustReplicateOptions{
+		meta:   meta,
+		status: replStatus,
+		opType: replication.ObjectReplicationType,
+	})
 }
 
 // mustReplicate returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
 // a synchronous manner.
-func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, metadataOnly bool) (replicate bool, sync bool) {
+func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, opts mustReplicateOptions) (replicate bool, sync bool) {
 	if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
 		return
 	}
-	return mustReplicater(ctx, bucket, object, meta, replStatus, metadataOnly)
+	return mustReplicater(ctx, bucket, object, opts)
 }
 
 // mustReplicater returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
 // a synchronous manner.
-func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string, metadataOnly bool) (replicate bool, sync bool) {
+func mustReplicater(ctx context.Context, bucket, object string, mopts mustReplicateOptions) (replicate bool, sync bool) {
 	if globalIsGateway {
 		return replicate, sync
 	}
-	if rs, ok := meta[xhttp.AmzBucketReplicationStatus]; ok {
+	replStatus := mopts.ReplicationStatus()
-		replStatus = rs
+	if replStatus == replication.Replica && !mopts.isMetadataReplication() {
-	}
-	if replication.StatusType(replStatus) == replication.Replica && !metadataOnly {
 		return replicate, sync
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)
@@ -124,11 +162,12 @@ func mustReplicater(ctx context.Context, bucket, object string, meta map[string]
 		return replicate, sync
 	}
 	opts := replication.ObjectOpts{
-		Name:    object,
+		Name:           object,
-		SSEC:    crypto.SSEC.IsEncrypted(meta),
+		SSEC:           crypto.SSEC.IsEncrypted(mopts.meta),
-		Replica: replication.StatusType(replStatus) == replication.Replica,
+		Replica:        replStatus == replication.Replica,
+		ExistingObject: mopts.isExistingObjectReplication(),
 	}
-	tagStr, ok := meta[xhttp.AmzObjectTagging]
+	tagStr, ok := mopts.meta[xhttp.AmzObjectTagging]
 	if ok {
 		opts.UserTags = tagStr
 	}
@@ -226,7 +265,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 // target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
 // deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
 // on target.
-func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectAPI ObjectLayer) {
+func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) {
 	bucket := dobj.Bucket
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
@@ -760,7 +799,9 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	if objInfo.UserTags != "" {
 		objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-
+	if ri.OpType == replication.ExistingObjectReplicationType {
+		objInfo.UserDefined[xhttp.MinIOReplicationResetStatus] = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), ri.ResetID)
+	}
 	// FIXME: add support for missing replication events
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
@@ -774,7 +815,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		return
 	}
 	// Leave metadata in `PENDING` state if inline replication fails to save iops
-	if ri.OpType == replication.HealReplicationType || replicationStatus == replication.Completed {
+	if ri.OpType == replication.HealReplicationType ||
+		replicationStatus == replication.Completed {
 		// This lower level implementation is necessary to avoid write locks from CopyObject.
 		poolIdx, err := z.getPoolIdx(ctx, bucket, object, objInfo.Size)
 		if err != nil {
@@ -834,10 +876,12 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 	return dst
 }
 
-// DeletedObjectVersionInfo has info on deleted object
+// DeletedObjectReplicationInfo has info on deleted object
-type DeletedObjectVersionInfo struct {
+type DeletedObjectReplicationInfo struct {
 	DeletedObject
-	Bucket string
+	Bucket  string
+	OpType  replication.Type
+	ResetID string
 }
 
 var (
@@ -847,35 +891,40 @@ var (
 
 // ReplicationPool describes replication pool
 type ReplicationPool struct {
-	objLayer        ObjectLayer
+	objLayer                ObjectLayer
-	ctx             context.Context
+	ctx                     context.Context
-	mrfWorkerKillCh chan struct{}
+	mrfWorkerKillCh         chan struct{}
-	workerKillCh    chan struct{}
+	workerKillCh            chan struct{}
-	replicaCh       chan ReplicateObjectInfo
+	replicaCh               chan ReplicateObjectInfo
-	replicaDeleteCh chan DeletedObjectVersionInfo
+	replicaDeleteCh         chan DeletedObjectReplicationInfo
-	mrfReplicaCh    chan ReplicateObjectInfo
+	mrfReplicaCh            chan ReplicateObjectInfo
-	workerSize      int
+	existingReplicaCh       chan ReplicateObjectInfo
-	mrfWorkerSize   int
+	existingReplicaDeleteCh chan DeletedObjectReplicationInfo
-	workerWg        sync.WaitGroup
+	workerSize              int
-	mrfWorkerWg     sync.WaitGroup
+	mrfWorkerSize           int
-	once            sync.Once
+	workerWg                sync.WaitGroup
-	mu              sync.Mutex
+	mrfWorkerWg             sync.WaitGroup
+	once                    sync.Once
+	mu                      sync.Mutex
 }
 
 // NewReplicationPool creates a pool of replication workers of specified size
 func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts) *ReplicationPool {
 	pool := &ReplicationPool{
-		replicaCh:       make(chan ReplicateObjectInfo, 100000),
+		replicaCh:               make(chan ReplicateObjectInfo, 100000),
-		replicaDeleteCh: make(chan DeletedObjectVersionInfo, 100000),
+		replicaDeleteCh:         make(chan DeletedObjectReplicationInfo, 100000),
-		mrfReplicaCh:    make(chan ReplicateObjectInfo, 100000),
+		mrfReplicaCh:            make(chan ReplicateObjectInfo, 100000),
-		workerKillCh:    make(chan struct{}, opts.Workers),
+		workerKillCh:            make(chan struct{}, opts.Workers),
-		mrfWorkerKillCh: make(chan struct{}, opts.FailedWorkers),
+		mrfWorkerKillCh:         make(chan struct{}, opts.FailedWorkers),
-		ctx:             ctx,
+		existingReplicaCh:       make(chan ReplicateObjectInfo, 100000),
-		objLayer:        o,
+		existingReplicaDeleteCh: make(chan DeletedObjectReplicationInfo, 100000),
+		ctx:                     ctx,
+		objLayer:                o,
 	}
 
 	pool.ResizeWorkers(opts.Workers)
 	pool.ResizeFailedWorkers(opts.FailedWorkers)
+	go pool.AddExistingObjectReplicateWorker()
 	return pool
 }
 
@@ -921,6 +970,26 @@ func (p *ReplicationPool) AddWorker() {
 
 }
 
+// AddExistingObjectReplicateWorker adds a worker to queue existing objects that need to be sync'd
+func (p *ReplicationPool) AddExistingObjectReplicateWorker() {
+	for {
+		select {
+		case <-p.ctx.Done():
+			return
+		case oi, ok := <-p.existingReplicaCh:
+			if !ok {
+				return
+			}
+			replicateObject(p.ctx, oi, p.objLayer)
+		case doi, ok := <-p.existingReplicaDeleteCh:
+			if !ok {
+				return
+			}
+			replicateDelete(p.ctx, doi, p.objLayer)
+		}
+	}
+}
+
 // ResizeWorkers sets replication workers pool to new size
 func (p *ReplicationPool) ResizeWorkers(n int) {
 	p.mu.Lock()
@@ -962,6 +1031,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		p.once.Do(func() {
 			close(p.replicaCh)
 			close(p.mrfReplicaCh)
+			close(p.existingReplicaCh)
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
@@ -972,27 +1042,44 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 	if p == nil {
 		return
 	}
+	var ch chan ReplicateObjectInfo
+	switch ri.OpType {
+	case replication.ExistingObjectReplicationType:
+		ch = p.existingReplicaCh
+	default:
+		ch = p.replicaCh
+	}
 	select {
 	case <-GlobalContext.Done():
 		p.once.Do(func() {
 			close(p.replicaCh)
 			close(p.mrfReplicaCh)
+			close(p.existingReplicaCh)
 		})
-	case p.replicaCh <- ri:
+	case ch <- ri:
 	default:
 	}
 }
 
-func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectVersionInfo) {
+func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInfo) {
 	if p == nil {
 		return
 	}
+	var ch chan DeletedObjectReplicationInfo
+	switch doi.OpType {
+	case replication.ExistingObjectReplicationType:
+		ch = p.existingReplicaDeleteCh
+	default:
+		ch = p.replicaDeleteCh
+	}
+
 	select {
 	case <-GlobalContext.Done():
 		p.once.Do(func() {
 			close(p.replicaDeleteCh)
+			close(p.existingReplicaDeleteCh)
 		})
-	case p.replicaDeleteCh <- doi:
+	case ch <- doi:
 	default:
 	}
 }
@@ -1157,7 +1244,78 @@ func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer,
 	}
 }
 
-func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectVersionInfo, o ObjectLayer, sync bool) {
+func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer, sync bool) {
 	globalReplicationPool.queueReplicaDeleteTask(dv)
 	globalReplicationStats.Update(dv.Bucket, 0, replication.Pending, replication.StatusType(""), replication.DeleteReplicationType)
 }
+
+type replicationConfig struct {
+	Config          *replication.Config
+	ResetID         string
+	ResetBeforeDate time.Time
+}
+
+func (c replicationConfig) Empty() bool {
+	return c.Config == nil
+}
+func (c replicationConfig) Replicate(opts replication.ObjectOpts) bool {
+	return c.Config.Replicate(opts)
+}
+
+// Resync returns true if replication reset is requested
+func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo) bool {
+	if c.Empty() {
+		return false
+	}
+	// existing object replication does not apply to un-versioned objects
+	if oi.VersionID == "" || oi.VersionID == nullVersionID {
+		return false
+	}
+
+	var replicate bool
+	if oi.DeleteMarker {
+		if c.Replicate(replication.ObjectOpts{
+			Name:           oi.Name,
+			SSEC:           crypto.SSEC.IsEncrypted(oi.UserDefined),
+			UserTags:       oi.UserTags,
+			DeleteMarker:   oi.DeleteMarker,
+			VersionID:      oi.VersionID,
+			OpType:         replication.DeleteReplicationType,
+			ExistingObject: true}) {
+			replicate = true
+		}
+	} else {
+		// Ignore previous replication status when deciding if object can be re-replicated
+		objInfo := oi.Clone()
+		objInfo.ReplicationStatus = replication.StatusType("")
+		replicate, _ = mustReplicater(ctx, oi.Bucket, oi.Name, getMustReplicateOptions(objInfo, replication.ExistingObjectReplicationType))
+	}
+	return c.resync(oi, replicate)
+}
+
+// wrapper function for testability. Returns true if a new reset is requested on
+// already replicated objects OR object qualifies for existing object replication
+// and no reset requested.
+func (c replicationConfig) resync(oi ObjectInfo, replicate bool) bool {
+	if !replicate {
+		return false
+	}
+	rs, ok := oi.UserDefined[xhttp.MinIOReplicationResetStatus]
+	if !ok { // existing object replication is enabled and object version is unreplicated so far.
+		if c.ResetID != "" && oi.ModTime.Before(c.ResetBeforeDate) { // trigger replication if `mc replicate reset` requested
+			return true
+		}
+		return oi.ReplicationStatus != replication.Completed
+	}
+	if c.ResetID == "" || c.ResetBeforeDate.Equal(timeSentinel) { // no reset in progress
+		return false
+	}
+	// if already replicated, return true if a new reset was requested.
+	splits := strings.SplitN(rs, ";", 2)
+	newReset := splits[1] != c.ResetID
+	if !newReset && oi.ReplicationStatus == replication.Completed {
+		// already replicated and no reset requested
+		return false
+	}
+	return newReset && oi.ModTime.Before(c.ResetBeforeDate)
+}

cmd/bucket-replication_test.go

@@ -0,0 +1,209 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+var configs = []replication.Config{
+	{ // Config0 - Replication config has no filters, existing object replication enabled
+		Rules: []replication.Rule{
+			{
+				Status:                    replication.Enabled,
+				Priority:                  1,
+				DeleteMarkerReplication:   replication.DeleteMarkerReplication{Status: replication.Enabled},
+				DeleteReplication:         replication.DeleteReplication{Status: replication.Enabled},
+				Filter:                    replication.Filter{},
+				ExistingObjectReplication: replication.ExistingObjectReplication{Status: replication.Enabled},
+				SourceSelectionCriteria: replication.SourceSelectionCriteria{
+					ReplicaModifications: replication.ReplicaModifications{Status: replication.Enabled},
+				},
+			},
+		},
+	},
+}
+
+var replicationConfigTests = []struct {
+	info         ObjectInfo
+	name         string
+	rcfg         replicationConfig
+	expectedSync bool
+}{
+	{ //1. no replication config
+		name:         "no replication config",
+		info:         ObjectInfo{Size: 100},
+		rcfg:         replicationConfig{Config: nil},
+		expectedSync: false,
+	},
+	{ //2. existing object replication config enabled, no versioning
+		name:         "existing object replication config enabled, no versioning",
+		info:         ObjectInfo{Size: 100},
+		rcfg:         replicationConfig{Config: &configs[0]},
+		expectedSync: false,
+	},
+	{ //3. existing object replication config enabled, versioning suspended
+		name:         "existing object replication config enabled, versioning suspended",
+		info:         ObjectInfo{Size: 100, VersionID: nullVersionID},
+		rcfg:         replicationConfig{Config: &configs[0]},
+		expectedSync: false,
+	},
+	{ //4. existing object replication enabled, versioning enabled; no reset in progress
+		name: "existing object replication enabled, versioning enabled; no reset in progress",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Completed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		rcfg:         replicationConfig{Config: &configs[0]},
+		expectedSync: false,
+	},
+}
+
+func TestReplicationResync(t *testing.T) {
+	ctx := context.Background()
+	for i, test := range replicationConfigTests {
+		if sync := test.rcfg.Resync(ctx, test.info); sync != test.expectedSync {
+			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync, test.expectedSync)
+		}
+	}
+}
+
+var start = UTCNow().AddDate(0, 0, -1)
+var replicationConfigTests2 = []struct {
+	info         ObjectInfo
+	name         string
+	replicate    bool
+	rcfg         replicationConfig
+	expectedSync bool
+}{
+	{ // Cases 1-4: existing object replication enabled, versioning enabled, no reset - replication status varies
+		// 1: Pending replication
+		name: "existing object replication on object in Pending replication status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Pending,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+	},
+	{ // 2. replication status Failed
+		name: "existing object replication on object in Failed replication status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Failed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+	},
+	{ //3. replication status unset
+		name: "existing object replication on pre-existing unreplicated object",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.StatusType(""),
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+	},
+	{ //4. replication status Complete
+		name: "existing object replication on object in Completed replication status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Completed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: false,
+	},
+	{ //5. existing object replication enabled, versioning enabled, replication status Pending & reset ID present
+		name: "existing object replication with reset in progress and object in Pending status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Pending,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+	{ //6. existing object replication enabled, versioning enabled, replication status Failed & reset ID present
+		name: "existing object replication with reset in progress and object in Failed status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Failed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+	{ //7. existing object replication enabled, versioning enabled, replication status unset & reset ID present
+		name: "existing object replication with reset in progress and object never replicated before",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.StatusType(""),
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+	{ //8. existing object replication enabled, versioning enabled, replication status Complete & reset ID present
+		name: "existing object replication enabled - reset in progress for an object in Completed status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Completed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df8",
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+	{ //9. existing object replication enabled, versioning enabled, replication status Pending & reset ID different
+		name: "existing object replication enabled, newer reset in progress on object in Pending replication status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Pending,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), "xyz")},
+			ModTime:           UTCNow().AddDate(0, 0, -1),
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "abc", ResetBeforeDate: UTCNow()},
+	},
+	{ //10. existing object replication enabled, versioning enabled, replication status Complete & reset done
+		name: "reset done on object in Completed Status - ineligbile for re-replication",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Completed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", start.Format(http.TimeFormat), "xyz")},
+		},
+		replicate:    true,
+		expectedSync: false,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+}
+
+func TestReplicationResyncwrapper(t *testing.T) {
+	for i, test := range replicationConfigTests2 {
+		if sync := test.rcfg.resync(test.info, test.replicate); sync != test.expectedSync {
+			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync, test.expectedSync)
+		}
+	}
+}

cmd/data-scanner.go

@@ -375,7 +375,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			activeLifeCycle = f.oldCache.Info.lifeCycle
 			filter = nil
 		}
-
+		// If there are replication rules for the prefix, remove the filter.
+		var replicationCfg replicationConfig
+		if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
+			replicationCfg = f.oldCache.Info.replication
+			filter = nil
+		}
 		// Check if we can skip it due to bloom filter...
 		if filter != nil && ok && existing.Compacted {
 			// If folder isn't in filter and we have data, skip it completely.
@@ -449,16 +454,16 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 
 			// Get file size, ignore errors.
 			item := scannerItem{
-				Path:       path.Join(f.root, entName),
+				Path:        path.Join(f.root, entName),
-				Typ:        typ,
+				Typ:         typ,
-				bucket:     bucket,
+				bucket:      bucket,
-				prefix:     path.Dir(prefix),
+				prefix:      path.Dir(prefix),
-				objectName: path.Base(entName),
+				objectName:  path.Base(entName),
-				debug:      f.dataUsageScannerDebug,
+				debug:       f.dataUsageScannerDebug,
-				lifeCycle:  activeLifeCycle,
+				lifeCycle:   activeLifeCycle,
-				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
+				replication: replicationCfg,
+				heal:        thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
-
 			// if the drive belongs to an erasure set
 			// that is already being healed, skip the
 			// healing attempt on this drive.
@@ -808,12 +813,13 @@ type scannerItem struct {
 	Path string
 	Typ  os.FileMode
 
-	bucket     string // Bucket.
+	bucket      string // Bucket.
-	prefix     string // Only the prefix if any, does not have final object name.
+	prefix      string // Only the prefix if any, does not have final object name.
-	objectName string // Only the object name without prefixes.
+	objectName  string // Only the object name without prefixes.
-	lifeCycle  *lifecycle.Lifecycle
+	lifeCycle   *lifecycle.Lifecycle
-	heal       bool // Has the object been selected for heal check?
+	replication replicationConfig
-	debug      bool
+	heal        bool // Has the object been selected for heal check?
+	debug       bool
 }
 
 type sizeSummary struct {
@@ -1140,33 +1146,50 @@ func (i *scannerItem) objectPath() string {
 
 // healReplication will heal a scanned item that has failed replication.
 func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
+	existingObjResync := i.replication.Resync(ctx, oi)
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		// heal delete marker replication failure or versioned delete replication failure
 		if oi.ReplicationStatus == replication.Pending ||
 			oi.ReplicationStatus == replication.Failed ||
 			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
-			i.healReplicationDeletes(ctx, o, oi)
+			i.healReplicationDeletes(ctx, o, oi, existingObjResync)
 			return
 		}
+		// if replication status is Complete on DeleteMarker and existing object resync required
+		if existingObjResync && oi.ReplicationStatus == replication.Completed {
+			i.healReplicationDeletes(ctx, o, oi, existingObjResync)
+			return
+		}
+		return
+	}
+	roi := ReplicateObjectInfo{ObjectInfo: oi, OpType: replication.HealReplicationType}
+	if existingObjResync {
+		roi.OpType = replication.ExistingObjectReplicationType
+		roi.ResetID = i.replication.ResetID
 	}
 	switch oi.ReplicationStatus {
 	case replication.Pending:
 		sizeS.pendingCount++
 		sizeS.pendingSize += oi.Size
-		globalReplicationPool.queueReplicaTask(ReplicateObjectInfo{ObjectInfo: oi, OpType: replication.HealReplicationType})
+		globalReplicationPool.queueReplicaTask(roi)
+		return
 	case replication.Failed:
 		sizeS.failedSize += oi.Size
 		sizeS.failedCount++
-		globalReplicationPool.queueReplicaTask(ReplicateObjectInfo{ObjectInfo: oi, OpType: replication.HealReplicationType})
+		globalReplicationPool.queueReplicaTask(roi)
+		return
 	case replication.Completed, "COMPLETE":
 		sizeS.replicatedSize += oi.Size
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
+	if existingObjResync {
+		globalReplicationPool.queueReplicaTask(roi)
+	}
 }
 
 // healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
+func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo, existingObject bool) {
 	// handle soft delete and permanent delete failures here.
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		versionID := ""
@@ -1176,7 +1199,7 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 		} else {
 			versionID = oi.VersionID
 		}
-		globalReplicationPool.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+		doi := DeletedObjectReplicationInfo{
 			DeletedObject: DeletedObject{
 				ObjectName:                    oi.Name,
 				DeleteMarkerVersionID:         dmVersionID,
@@ -1187,7 +1210,12 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 				VersionPurgeStatus:            oi.VersionPurgeStatus,
 			},
 			Bucket: oi.Bucket,
-		})
+		}
+		if existingObject {
+			doi.OpType = replication.ExistingObjectReplicationType
+			doi.ResetID = i.replication.ResetID
+		}
+		globalReplicationPool.queueReplicaDeleteTask(doi)
 	}
 }
 

cmd/data-usage-cache.go

@@ -161,7 +161,8 @@ type dataUsageCacheInfo struct {
 	// optional updates channel.
 	// If set updates will be sent regularly to this channel.
 	// Will not be closed when returned.
-	updates chan<- dataUsageEntry `msg:"-"`
+	updates     chan<- dataUsageEntry `msg:"-"`
+	replication replicationConfig     `msg:"-"`
 }
 
 func (e *dataUsageEntry) addSizes(summary sizeSummary) {

cmd/object-api-datatypes.go

@@ -230,6 +230,7 @@ type ReplicateObjectInfo struct {
 	ObjectInfo
 	OpType     replication.Type
 	RetryCount uint32
+	ResetID    string
 }
 
 // MultipartInfo captures metadata information about the uploadId

cmd/object-handlers.go

@@ -1291,7 +1291,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	if rs := r.Header.Get(xhttp.AmzBucketReplicationStatus); rs != "" {
 		srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = rs
 	}
-	if ok, _ := mustReplicate(ctx, r, dstBucket, dstObject, srcInfo.UserDefined, srcInfo.ReplicationStatus.String(), srcInfo.metadataOnly); ok {
+	if ok, _ := mustReplicate(ctx, r, dstBucket, dstObject, getMustReplicateOptions(srcInfo, replication.UnsetReplicationType)); ok {
 		srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 	// Store the preserved compression metadata.
@@ -1393,7 +1393,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
 	response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
 	encodedSuccessResponse := encodeResponse(response)
-	if replicate, sync := mustReplicate(ctx, r, dstBucket, dstObject, objInfo.UserDefined, objInfo.ReplicationStatus.String(), objInfo.metadataOnly); replicate {
+
+	if replicate, sync := mustReplicate(ctx, r, dstBucket, dstObject, getMustReplicateOptions(objInfo, replication.UnsetReplicationType)); replicate {
 		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync, replication.ObjectReplicationType)
 	}
 
@@ -1637,7 +1638,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if ok, _ := mustReplicate(ctx, r, bucket, object, metadata, "", false); ok {
+	if ok, _ := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(ObjectInfo{
+		UserDefined: metadata,
+	}, replication.ObjectReplicationType)); ok {
 		metadata[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 	if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
@@ -1720,7 +1723,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			}
 		}
 	}
-	if replicate, sync := mustReplicate(ctx, r, bucket, object, metadata, "", false); replicate {
+	if replicate, sync := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(ObjectInfo{
+		UserDefined: metadata,
+	}, replication.ObjectReplicationType)); replicate {
 		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync, replication.ObjectReplicationType)
 	}
 
@@ -1956,7 +1961,9 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			return
 		}
 
-		if ok, _ := mustReplicate(ctx, r, bucket, object, metadata, "", false); ok {
+		if ok, _ := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(ObjectInfo{
+			UserDefined: metadata,
+		}, replication.ObjectReplicationType)); ok {
 			metadata[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 		}
 
@@ -2012,7 +2019,9 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			return
 		}
 
-		if replicate, sync := mustReplicate(ctx, r, bucket, object, metadata, "", false); replicate {
+		if replicate, sync := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(ObjectInfo{
+			UserDefined: metadata,
+		}, replication.ObjectReplicationType)); replicate {
 			scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync, replication.ObjectReplicationType)
 
 		}
@@ -2124,7 +2133,9 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if ok, _ := mustReplicate(ctx, r, bucket, object, metadata, "", false); ok {
+	if ok, _ := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(ObjectInfo{
+		UserDefined: metadata,
+	}, replication.ObjectReplicationType)); ok {
 		metadata[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 	// We need to preserve the encryption headers set in EncryptRequest,
@@ -3114,7 +3125,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	setPutObjHeaders(w, objInfo, false)
-	if replicate, sync := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String(), false); replicate {
+	if replicate, sync := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(objInfo, replication.ObjectReplicationType)); replicate {
 		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync, replication.ObjectReplicationType)
 	}
 
@@ -3288,7 +3299,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		} else {
 			versionID = objInfo.VersionID
 		}
-		dobj := DeletedObjectVersionInfo{
+		dobj := DeletedObjectReplicationInfo{
 			DeletedObject: DeletedObject{
 				ObjectName:                    object,
 				VersionID:                     versionID,
@@ -3375,7 +3386,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 	objInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status))
-	replicate, sync := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String(), true)
+	replicate, sync := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(objInfo, replication.MetadataReplicationType))
 	if replicate {
 		objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
@@ -3554,7 +3565,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		objInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = ""
 		objInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = ""
 	}
-	replicate, sync := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String(), true)
+	replicate, sync := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(objInfo, replication.MetadataReplicationType))
 	if replicate {
 		objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
@@ -3751,14 +3762,16 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	replicate, sync := mustReplicate(ctx, r, bucket, object, map[string]string{xhttp.AmzObjectTagging: tags.String()}, objInfo.ReplicationStatus.String(), true)
+	tagsStr := tags.String()
+
+	oi := objInfo.Clone()
+	oi.UserTags = tagsStr
+	replicate, sync := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(oi, replication.MetadataReplicationType))
 	if replicate {
 		opts.UserDefined = make(map[string]string)
 		opts.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 
-	tagsStr := tags.String()
-
 	// Put object tags
 	objInfo, err = objAPI.PutObjectTags(ctx, bucket, object, tagsStr, opts)
 	if err != nil {
@@ -3828,7 +3841,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	replicate, sync := mustReplicate(ctx, r, bucket, object, map[string]string{xhttp.AmzObjectTagging: oi.UserTags}, oi.ReplicationStatus.String(), true)
+	replicate, sync := mustReplicate(ctx, r, bucket, object, getMustReplicateOptions(oi, replication.MetadataReplicationType))
 	if replicate {
 		opts.UserDefined = make(map[string]string)
 		opts.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()

cmd/web-handlers.go

@@ -814,7 +814,7 @@ next:
 			})
 
 			if replicateDel {
-				dobj := DeletedObjectVersionInfo{
+				dobj := DeletedObjectReplicationInfo{
 					DeletedObject: DeletedObject{
 						ObjectName:                    objectName,
 						DeleteMarkerVersionID:         oi.VersionID,
@@ -948,7 +948,7 @@ next:
 					Host:       sourceIP,
 				})
 				if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
-					dv := DeletedObjectVersionInfo{
+					dv := DeletedObjectReplicationInfo{
 						DeletedObject: dobj,
 						Bucket:        args.BucketName,
 					}
@@ -1256,7 +1256,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	mustReplicate, sync := mustReplicateWeb(ctx, r, bucket, object, metadata, "", replPerms)
+	mustReplicate, sync := mustReplicateWeb(ctx, r, bucket, object, metadata, replication.StatusType(""), replPerms)
 	if mustReplicate {
 		metadata[xhttp.AmzBucketReplicationStatus] = string(replication.Pending)
 	}

cmd/xl-storage.go

@@ -414,6 +414,19 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 		}
 	}
 
+	// Check if the current bucket has replication configuration
+	if rcfg, err := globalBucketMetadataSys.GetReplicationConfig(ctx, cache.Info.Name); err == nil {
+		if rcfg.HasActiveRules("", true) {
+			tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, cache.Info.Name, rcfg.RoleArn)
+			cache.Info.replication = replicationConfig{
+				Config:          rcfg,
+				ResetID:         tgt.ResetID,
+				ResetBeforeDate: tgt.ResetBeforeDate}
+			if intDataUpdateTracker.debug {
+				console.Debugln(color.Green("scannerDisk:") + " replication: Active rules found")
+			}
+		}
+	}
 	// return initialized object layer
 	objAPI := newObjectLayerFn()
 	// object layer not initialized, return.
@@ -452,7 +465,6 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 			}
 			return sizeSummary{}, errSkipFile
 		}
-
 		sizeS := sizeSummary{}
 		for _, version := range fivs.Versions {
 			oi := version.ToObjectInfo(item.bucket, item.objectPath())

docs/bucket/replication/DESIGN.md

@@ -33,7 +33,11 @@ An additional header `X-Minio-Replication-Delete-Status` is returned which would
 
 Note that synchronous replication, i.e. when remote target is configured with --sync mode in `mc admin bucket remote add` does not apply to `DELETE` operations. The version being deleted on the source cluster needs to maintain state and ensure that the operation is mirrored to the target cluster prior to completing on the source object version. Since this needs to account for the target cluster availability and the need to serialize concurrent DELETE operations on different versions of the same object during multi DELETE operations, the current implementation queues the `DELETE` operations in both sync and async modes.
 
-Existing object replication, replica modification sync for 2-way replication and multi site replication are currently not supported.
+### Existing object replication
+Existing object replication works similar to regular replication. Objects qualifying for existing object replication are detected when scanner runs, and will be replicated if existing object replication is enabled and applicable replication rules are satisfied. Because replication depends on the immutability of versions, only pre-existing objects created while versioning was enabled can be replicated. Even if replication rules are disabled and re-enabled later, the objects created during the interim will be synced as the scanner queues them. For saving iops, objects qualifying for
+existing object replication are not marked as `PENDING` prior to replication.
+
+If the remote site is fully lost and objects previously replicated need to be re-synced, the `mc replicate reset` command with optional flag of `--older-than` needs to be used to trigger re-syncing of previously replicated objects. This command generates a ResetID which is a unique UUID saved to the remote target config along with the applicable date(defaults to time of initiating the reset). All objects created prior to this date are eligible for re-replication if existing object replication is enabled for the replication rule the object satisifes. At the time of completion of replication, `X-Minio-Replication-Reset-Status` is set in the metadata with the timestamp of replication and ResetID. For saving iops, the objects which are re-replicated are not first set to `PENDING` state.
 
 ### Internal metadata for replication
 

docs/bucket/replication/README.md

@@ -202,6 +202,21 @@ remote replication target using the `mc admin bucket remote add` command
  mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1 --sync --healthcheck-seconds 100
 ```
 
+### Existing object replication
+Existing object replication as detailed [here](https://aws.amazon.com/blogs/storage/replicating-existing-objects-between-s3-buckets/) can be enabled by passing `existing-objects` as a value to `--replicate` flag while adding or editing a replication rule.
+
+Once existing object replication is enabled, all objects or object prefixes that satisfy the replication rules and were created prior to adding replication configuration OR while replication rules were disabled will be synced to the target cluster. Depending on the number of previously existing objects, the existing objects that are now eligible to be replicated will eventually be synced to the target cluster as the scanner schedules them. This may be slower depending on the load on the cluster, latency and size of the namespace.
+
+Note that existing object replication requires that the objects being replicated were created while versioning was enabled. Objects with null version (i.e. those created when versioning is not enabled or versioning is suspended) will not be replicated.
+
+In the rare event that target DR site is entirely lost and previously replicated objects to the DR cluster need to be re-replicated, `mc replicate reset alias/bucket` can be used to initiate a reset. This would initiate a re-sync between the two clusters on a lower priority as the scanner picks up these objects to re-sync.
+
+This is an expensive operation and should be initiated only once - progress of the syncing can be monitored by looking at Prometheus metrics. If object version has been re-replicated, `mc stat --vid --debug` on this version shows an additional header `X-Minio-Replication-Reset-Status` with the replication timestamp and ResetID generated at the time of issuing the `mc replicate reset` command.
+
+Note that ExistingObjectReplication needs to be enabled in the config via `mc replicate [add|edit]` by passing `existing-objects` as one of the values to `--replicate` flag. Only those objects meeting replication rules and having existing object replication enabled will be re-synced.
+
+Multi site replication is currently not supported.
+
 ## Explore Further
 - [MinIO Bucket Replication Design](https://github.com/minio/minio/blob/master/docs/bucket/replication/DESIGN.md)
 - [MinIO Bucket Versioning Implementation](https://docs.minio.io/docs/minio-bucket-versioning-guide.html)

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/minio/madmin-go v1.0.9
 	github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78
 	github.com/minio/parquet-go v1.0.0
-	github.com/minio/pkg v1.0.3
+	github.com/minio/pkg v1.0.4
 	github.com/minio/rpc v1.0.0
 	github.com/minio/selfupdate v0.3.1
 	github.com/minio/sha256-simd v1.0.0

go.sum

@@ -494,6 +494,8 @@ github.com/minio/parquet-go v1.0.0 h1:fcWsEvub04Nsl/4hiRBDWlbqd6jhacQieV07a+nhiI
 github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTacxxk/oHA=
 github.com/minio/pkg v1.0.3 h1:tUhM6lG/BdNB0+5f2RbE4ifCAYwMs6cRJnZ/AY0WIeQ=
 github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
+github.com/minio/pkg v1.0.4 h1:+BmaCENP6BaMm9PsGK6L1L5MKulWDxl4qobvJYf6m/E=
+github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/rpc v1.0.0 h1:tJCHyLfQF6k6HlMQFpKy2FO/7lc2WP8gLDGMZp18E70=
 github.com/minio/rpc v1.0.0/go.mod h1:b9xqF7J0xeMXr0cM4pnBlP7Te7PDsG5JrRxl5dG6Ldk=
 github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=

internal/bucket/replication/replication.go

@@ -131,23 +131,31 @@ type Type int
 
 // Types of replication
 const (
-	ObjectReplicationType Type = 1 + iota
+	UnsetReplicationType Type = 0 + iota
+	ObjectReplicationType
 	DeleteReplicationType
 	MetadataReplicationType
 	HealReplicationType
+	ExistingObjectReplicationType
 )
 
+// Valid returns true if replication type is set
+func (t Type) Valid() bool {
+	return t > 0
+}
+
 // ObjectOpts provides information to deduce whether replication
 // can be triggered on the resultant object.
 type ObjectOpts struct {
-	Name         string
+	Name           string
-	UserTags     string
+	UserTags       string
-	VersionID    string
+	VersionID      string
-	IsLatest     bool
+	IsLatest       bool
-	DeleteMarker bool
+	DeleteMarker   bool
-	SSEC         bool
+	SSEC           bool
-	OpType       Type
+	OpType         Type
-	Replica      bool
+	Replica        bool
+	ExistingObject bool
 }
 
 // FilterActionableRules returns the rules actions that need to be executed
@@ -191,6 +199,9 @@ func (c Config) Replicate(obj ObjectOpts) bool {
 		if rule.Status == Disabled {
 			continue
 		}
+		if obj.ExistingObject && rule.ExistingObjectReplication.Status == Disabled {
+			return false
+		}
 		if obj.OpType == DeleteReplicationType {
 			switch {
 			case obj.VersionID != "":

internal/bucket/replication/rule.go

@@ -90,6 +90,43 @@ func (d *DeleteReplication) UnmarshalXML(dec *xml.Decoder, start xml.StartElemen
 	return nil
 }
 
+// ExistingObjectReplication - whether existing object replication is enabled
+type ExistingObjectReplication struct {
+	Status Status `xml:"Status"` // should be set to "Disabled" by default
+}
+
+// IsEmpty returns true if ExistingObjectReplication is not set
+func (e ExistingObjectReplication) IsEmpty() bool {
+	return len(e.Status) == 0
+}
+
+// Validate validates whether the status is disabled.
+func (e ExistingObjectReplication) Validate() error {
+	if e.IsEmpty() {
+		return nil
+	}
+	if e.Status != Disabled && e.Status != Enabled {
+		return errInvalidExistingObjectReplicationStatus
+	}
+	return nil
+}
+
+// UnmarshalXML - decodes XML data. Default to Disabled unless specified
+func (e *ExistingObjectReplication) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) {
+	// Make subtype to avoid recursive UnmarshalXML().
+	type existingObjectReplication ExistingObjectReplication
+	erep := existingObjectReplication{}
+
+	if err := dec.DecodeElement(&erep, &start); err != nil {
+		return err
+	}
+	if len(erep.Status) == 0 {
+		erep.Status = Disabled
+	}
+	e.Status = erep.Status
+	return nil
+}
+
 // Rule - a rule for replication configuration.
 type Rule struct {
 	XMLName                 xml.Name                `xml:"Rule" json:"Rule"`
@@ -98,22 +135,24 @@ type Rule struct {
 	Priority                int                     `xml:"Priority" json:"Priority"`
 	DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication" json:"DeleteMarkerReplication"`
 	// MinIO extension to replicate versioned deletes
-	DeleteReplication       DeleteReplication       `xml:"DeleteReplication" json:"DeleteReplication"`
+	DeleteReplication         DeleteReplication         `xml:"DeleteReplication" json:"DeleteReplication"`
-	Destination             Destination             `xml:"Destination" json:"Destination"`
+	Destination               Destination               `xml:"Destination" json:"Destination"`
-	SourceSelectionCriteria SourceSelectionCriteria `xml:"SourceSelectionCriteria" json:"SourceSelectionCriteria"`
+	SourceSelectionCriteria   SourceSelectionCriteria   `xml:"SourceSelectionCriteria" json:"SourceSelectionCriteria"`
-	Filter                  Filter                  `xml:"Filter" json:"Filter"`
+	Filter                    Filter                    `xml:"Filter" json:"Filter"`
+	ExistingObjectReplication ExistingObjectReplication `xml:"ExistingObjectReplication,omitempty" json:"ExistingObjectReplication,omitempty"`
 }
 
 var (
-	errInvalidRuleID                        = Errorf("ID must be less than 255 characters")
+	errInvalidRuleID                          = Errorf("ID must be less than 255 characters")
-	errEmptyRuleStatus                      = Errorf("Status should not be empty")
+	errEmptyRuleStatus                        = Errorf("Status should not be empty")
-	errInvalidRuleStatus                    = Errorf("Status must be set to either Enabled or Disabled")
+	errInvalidRuleStatus                      = Errorf("Status must be set to either Enabled or Disabled")
-	errDeleteMarkerReplicationMissing       = Errorf("DeleteMarkerReplication must be specified")
+	errDeleteMarkerReplicationMissing         = Errorf("DeleteMarkerReplication must be specified")
-	errPriorityMissing                      = Errorf("Priority must be specified")
+	errPriorityMissing                        = Errorf("Priority must be specified")
-	errInvalidDeleteMarkerReplicationStatus = Errorf("Delete marker replication status is invalid")
+	errInvalidDeleteMarkerReplicationStatus   = Errorf("Delete marker replication status is invalid")
-	errDestinationSourceIdentical           = Errorf("Destination bucket cannot be the same as the source bucket.")
+	errDestinationSourceIdentical             = Errorf("Destination bucket cannot be the same as the source bucket.")
-	errDeleteReplicationMissing             = Errorf("Delete replication must be specified")
+	errDeleteReplicationMissing               = Errorf("Delete replication must be specified")
-	errInvalidDeleteReplicationStatus       = Errorf("Delete replication is either enable|disable")
+	errInvalidDeleteReplicationStatus         = Errorf("Delete replication is either enable|disable")
+	errInvalidExistingObjectReplicationStatus = Errorf("Existing object replication status is invalid")
 )
 
 // validateID - checks if ID is valid or not.
@@ -200,7 +239,7 @@ func (r Rule) Validate(bucket string, sameTarget bool) error {
 	if r.Destination.Bucket == bucket && sameTarget {
 		return errDestinationSourceIdentical
 	}
-	return nil
+	return r.ExistingObjectReplication.Validate()
 }
 
 // MetadataReplicate  returns true if object is not a replica or in the case of replicas,

internal/http/headers.go

@@ -170,6 +170,9 @@ const (
 	MinIOSourceProxyRequest = "X-Minio-Source-Proxy-Request"
 	// Header indicates that this request is a replication request to create a REPLICA
 	MinIOSourceReplicationRequest = "X-Minio-Source-Replication-Request"
+	// Header indicates replication reset status.
+	MinIOReplicationResetStatus = "X-Minio-Replication-Reset-Status"
+
 	// predicted date/time of transition
 	MinIOTransition = "X-Minio-Transition"
 )