feat: add idempotent delete marker support (#15521)

The bottom line is delete markers are a nuisance,
most applications are not version aware and this
has simply complicated the version management.

AWS S3 gave an unnecessary complication overhead
for customers, they need to now manage these
markers by applying ILM settings and clean
them up on a regular basis.

To make matters worse all these delete markers
get replicated as well in a replicated setup,
requiring two ILM settings on each site.

This PR is an attempt to address this inferior
implementation by deviating MinIO towards an
idempotent delete marker implementation i.e
MinIO will never create any more than single
consecutive delete markers.

This significantly reduces operational overhead
by making versioning more useful for real data.

This is an S3 spec deviation for pragmatic reasons.

docs/bucket/replication/setup_2site_existing_replication.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -x
+
 trap 'catch $LINENO' ERR
 
 # shellcheck disable=SC2120
@@ -37,8 +39,10 @@ unset MINIO_KMS_KES_KEY_FILE
 unset MINIO_KMS_KES_ENDPOINT
 unset MINIO_KMS_KES_KEY_NAME
 
-wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \
-    && chmod +x mc
+if [ ! -f ./mc ]; then
+    wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
+        chmod +x mc
+fi
 
 minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
       "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
@@ -66,6 +70,9 @@ done
 ./mc mirror /tmp/data sitea/bucket/
 ./mc version enable sitea/bucket
 
+./mc cp /tmp/data/file_1.txt sitea/bucket/marker
+./mc rm sitea/bucket/marker
+
 ./mc mb siteb/bucket/
 ./mc version enable siteb/bucket/
 
@@ -83,11 +90,49 @@ sleep 1
 sleep 10s ## sleep for 10s idea is that we give 100ms per object.
 
 count=$(./mc replicate resync status sitea/bucket --remote-bucket "${remote_arn}" --json | jq .resyncInfo.target[].replicationCount)
-./mc ls --versions sitea/bucket
-./mc ls --versions siteb/bucket
-if [ $count -ne 10 ]; then
-    echo "resync not complete after 100s unexpected failure"
+
+./mc ls -r --versions sitea/bucket > /tmp/sitea.txt
+./mc ls -r --versions siteb/bucket > /tmp/siteb.txt
+
+out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after replication: $out"
+    exit 1
+fi
+
+if [ $count -ne 12 ]; then
+    echo "resync not complete after 10s unexpected failure"
     ./mc diff sitea/bucket siteb/bucket
+    exit 1
+fi
+
+./mc cp /tmp/data/file_1.txt sitea/bucket/marker_new
+./mc rm sitea/bucket/marker_new
+
+sleep 12s ## sleep for 12s idea is that we give 100ms per object.
+
+./mc ls -r --versions sitea/bucket > /tmp/sitea.txt
+./mc ls -r --versions siteb/bucket > /tmp/siteb.txt
+
+out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no 'diff' after replication: $out"
+    exit 1
+fi
+
+./mc rm -r --force --versions sitea/bucket/marker
+sleep 14s ## sleep for 14s idea is that we give 100ms per object.
+
+./mc ls -r --versions sitea/bucket > /tmp/sitea.txt
+./mc ls -r --versions siteb/bucket > /tmp/siteb.txt
+
+out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no 'diff' after replication: $out"
+    exit 1
 fi
 
 catch

docs/bucket/replication/setup_3site_replication.sh

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+if [ -n "$TEST_DEBUG" ]; then
+    set -x
+fi
+
 trap 'catch $LINENO' ERR
 
 # shellcheck disable=SC2120