attempt delete marker replication after object is replicated (#15857)

Ensure delete marker replication success, especially since the
recent optimizations to heal on HEAD, LIST and GET can force
replication attempts on delete marker before underlying object
version could have synced.

cmd/bucket-replication.go

@@ -568,18 +568,26 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		return
 	}
 	// early return if already replicated delete marker for existing object replication/ healing delete markers
-	if dobj.DeleteMarkerVersionID != "" && (dobj.OpType == replication.ExistingObjectReplicationType || dobj.OpType == replication.HealReplicationType) {
+	if dobj.DeleteMarkerVersionID != "" {
-		if _, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, miniogo.StatObjectOptions{
+		toi, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, miniogo.StatObjectOptions{
 			VersionID: versionID,
 			Internal: miniogo.AdvancedGetOptions{
 				ReplicationProxyRequest: "false",
+				ReplicationDeleteMarker: true,
 			},
-		}); isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
+		})
+		if isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
 			if dobj.VersionID == "" {
 				rinfo.ReplicationStatus = replication.Completed
 				return
 			}
 		}
+		// mark delete marker replication as failed if target cluster not ready to receive
+		// this request yet (object version not replicated yet)
+		if err != nil && !toi.ReplicationReady {
+			rinfo.ReplicationStatus = replication.Failed
+			return
+		}
 	}
 
 	rmErr := tgt.RemoveObject(ctx, tgt.Bucket, dobj.ObjectName, miniogo.RemoveObjectOptions{
@@ -1897,7 +1905,9 @@ func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 	if opts.VersionSuspended {
 		return &madmin.BucketTargets{}
 	}
-
+	if !opts.ProxyRequest {
+		return &madmin.BucketTargets{}
+	}
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || cfg == nil {
 		return &madmin.BucketTargets{}

cmd/object-handlers.go

@@ -740,6 +740,15 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 			}
 			QueueReplicationHeal(ctx, bucket, objInfo)
 		}
+		// do an additional verification whether object exists when opts.DeleteMarker is set by source
+		// cluster as part of delete marker replication
+		if opts.DeleteMarker && opts.ProxyHeaderSet {
+			opts.VersionID = ""
+			goi, gerr := getObjectInfo(ctx, bucket, object, opts)
+			if gerr == nil || goi.VersionID != "" { // object layer returned more info because object is deleted
+				w.Header().Set(xhttp.MinIOTargetReplicationReady, "true")
+			}
+		}
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}

go.mod

@@ -49,7 +49,7 @@ require (
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.21.0
 	github.com/minio/madmin-go v1.6.2
-	github.com/minio/minio-go/v7 v7.0.40-0.20220928095841-8848d8affe8a
+	github.com/minio/minio-go/v7 v7.0.41-0.20221013203648-8257e7003b5e
 	github.com/minio/pkg v1.5.2
 	github.com/minio/selfupdate v0.5.0
 	github.com/minio/sha256-simd v1.0.0

go.sum

@@ -661,6 +661,8 @@ github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEp
 github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
 github.com/minio/minio-go/v7 v7.0.40-0.20220928095841-8848d8affe8a h1:COFh7S3tOKmJNYtKKFAuHQFH7MAaXxg4aAluXC9KQgc=
 github.com/minio/minio-go/v7 v7.0.40-0.20220928095841-8848d8affe8a/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
+github.com/minio/minio-go/v7 v7.0.41-0.20221013203648-8257e7003b5e h1:Sq8FAHvwrz9x+Nwb1dt9f/BSmje07a6Q5LXdmprff/A=
+github.com/minio/minio-go/v7 v7.0.41-0.20221013203648-8257e7003b5e/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
 github.com/minio/pkg v1.1.20/go.mod h1:Xo7LQshlxGa9shKwJ7NzQbgW4s8T/Wc1cOStR/eUiMY=
 github.com/minio/pkg v1.5.2 h1:vyEZ3TroiRGS/qb1XgP9RpK2zhjSWpBPjhNEbIo0pY8=
 github.com/minio/pkg v1.5.2/go.mod h1:koF2J2Ep/zpd//k+3UYdh6ySZKjqzy9C6RCZRX7uRY8=

internal/http/headers.go

@@ -187,7 +187,8 @@ const (
 	MinIOSourceReplicationRequest = "X-Minio-Source-Replication-Request"
 	// Header indicates replication reset status.
 	MinIOReplicationResetStatus = "X-Minio-Replication-Reset-Status"
-
+	// Header indicating target cluster can receive delete marker replication requests because object has been replicated
+	MinIOTargetReplicationReady = "X-Minio-Replication-Ready"
 	// Header indiicates last tag update time on source
 	MinIOSourceTaggingTimestamp = "X-Minio-Source-Replication-Tagging-Timestamp"
 	// Header indiicates last rtention update time on source