site replication: fix healing of bucket deletes. (#15377)

This PR changes the handling of bucket deletes for site 
replicated setups to hold on to deleted bucket state until 
it syncs to all the clusters participating in site replication.

docs/site-replication/run-multi-site-ldap.sh

@@ -142,6 +142,9 @@ fi
 
 ./mc mb minio1/newbucket
 
+# create a bucket bucket2 on minio1.
+./mc mb minio1/bucket2
+
 sleep 5
 ./mc stat minio2/newbucket
 if [ $? -ne 0 ]; then
@@ -246,7 +249,8 @@ if [ $? -ne 0 ]; then
     echo "expecting tag set to be successful. exiting.."
     exit_1;
 fi
-sleep 5
+
+sleep 10
 
 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key)
 if [ "${val}" != "val1" ]; then
@@ -259,10 +263,12 @@ kill -9 ${site1_pid}
 ./mc tag set minio2/newbucket "key=val2"
 # create a new bucket on minio2. This should replicate to minio1 after it comes online.
 ./mc mb minio2/newbucket2
+# delete bucket2 on minio2. This should replicate to minio1 after it comes online.
+./mc rb minio2/bucket2
 
 # Restart minio1 instance
 minio server --config-dir /tmp/minio-ldap --address ":9001" /tmp/minio-ldap-idp1/{1...4} >/tmp/minio1_1.log 2>&1 &
-sleep 10
+sleep 15
 # Test whether most recent tag update on minio2 is replicated to minio1
 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key )
 if [ "${val}" != "val2" ]; then
@@ -270,10 +276,11 @@ if [ "${val}" != "val2" ]; then
     exit_1;
 fi
 
-# Test if bucket created when minio1 is down healed
-diff -q <(./mc ls minio1 | awk '{print $3}') <(./mc ls minio2 | awk '{print $3}') 1>/dev/null
+# Test if bucket created/deleted when minio1 is down healed
+diff -q <(./mc ls minio1)  <(./mc ls minio2) 1>/dev/null
 if  [ $? -ne 0 ]; then
-    echo "expected bucket to have replicated, exiting..."
+    echo "expected `bucket2` delete and `newbucket2` creation to have replicated, exiting..."
     exit_1;
 fi
+
 cleanup

docs/site-replication/run-multi-site-minio-idp.sh

@@ -305,6 +305,9 @@ if [ $? -ne 0 ]; then
     exit_1;
 fi
 
+# create a bucket bucket2 on minio1.
+./mc mb minio1/bucket2
+
 sleep 10
 
 # Test whether policy unset replicated to minio1
@@ -319,6 +322,9 @@ kill -9 ${site1_pid}
 ./mc tag set minio2/newbucket "key=val2"
 # create a new bucket on minio2. This should replicate to minio1 after it comes online.
 ./mc mb minio2/newbucket2
+
+# delete bucket2 on minio2. This should replicate to minio1 after it comes online.
+./mc rb minio2/bucket2
 # Restart minio1 instance
 minio server --config-dir /tmp/minio-internal --address ":9001" /tmp/minio-internal-idp1/{1...4} >/tmp/minio1_1.log 2>&1 &
 sleep 15
@@ -329,9 +335,9 @@ if [ "${val}" != "val2" ]; then
     echo "expected bucket tag to have replicated, exiting..."
     exit_1;
 fi
-# Test if bucket created when minio1 is down healed
-diff -q <(./mc ls minio1 | awk '{print $3}') <(./mc ls minio2 | awk '{print $3}') 1>/dev/null
+# Test if bucket created/deleted when minio1 is down healed
+diff -q <(./mc ls minio1)  <(./mc ls minio2) 1>/dev/null
 if  [ $? -ne 0 ]; then
-    echo "expected bucket to have replicated, exiting..."
+    echo "expected `bucket2` delete and `newbucket2` creation to have replicated, exiting..."
     exit_1;
 fi

docs/site-replication/run-multi-site-oidc.sh

@@ -155,6 +155,9 @@ if [ $? -eq 0 ]; then
     exit_1;
 fi
 
+# create a bucket bucket2 on minio1.
+./mc mb minio1/bucket2
+
 ./mc mb minio1/newbucket
 
 sleep 5
@@ -231,8 +234,8 @@ if [ $? -ne 0 ]; then
     echo "expecting tag set to be successful. exiting.."
     exit_1;
 fi
-sleep 5
 
+sleep 10
 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key)
 if [ "${val}" != "val1" ]; then
     echo "expected bucket tag to have replicated, exiting..."
@@ -244,10 +247,12 @@ kill -9 ${site1_pid}
 ./mc tag set minio2/newbucket "key=val2"
 # create a new bucket on minio2. This should replicate to minio1 after it comes online.
 ./mc mb minio2/newbucket2
+# delete bucket2 on minio2. This should replicate to minio1 after it comes online.
+./mc rb minio2/bucket2
 
 # Restart minio1 instance
 minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 &
-sleep 10
+sleep 15
 # Test whether most recent tag update on minio2 is replicated to minio1
 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key )
 if [ "${val}" != "val2" ]; then
@@ -255,9 +260,9 @@ if [ "${val}" != "val2" ]; then
     exit_1;
 fi
 
-# Test if bucket created when minio1 is down healed
-diff -q <(./mc ls minio1 | awk '{print $3}') <(./mc ls minio2 | awk '{print $3}') 1>/dev/null
+# Test if bucket created/deleted when minio1 is down healed
+diff -q <(./mc ls minio1)  <(./mc ls minio2) 1>/dev/null
 if  [ $? -ne 0 ]; then
-    echo "expected bucket to have replicated, exiting..."
+    echo "expected `bucket2` delete and `newbucket2` creation to have replicated, exiting..."
     exit_1;
 fi