From f492f7215432fddba80e407e8b3ea7f7be8f67df Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Mon, 20 Sep 2021 18:24:24 -0700
Subject: [PATCH] add 3site replication script (#13256)

---
 docs/bucket/replication/README.md             |   2 +-
 .../replication/setup_3site_replication.sh    | 185 ++++++++++++++++++
 docs/bucket/replication/setup_replication.sh  |  15 +-
 3 files changed, 193 insertions(+), 9 deletions(-)
 create mode 100755 docs/bucket/replication/setup_3site_replication.sh

diff --git a/docs/bucket/replication/README.md b/docs/bucket/replication/README.md
index d8a343d65..eab07905a 100644
--- a/docs/bucket/replication/README.md
+++ b/docs/bucket/replication/README.md
@@ -218,7 +218,7 @@ Note that ExistingObjectReplication needs to be enabled in the config via `mc re
 
 ### Multi destination replication
 
-Replication from a source bucket to multiple destination buckets is supported. For each of the targets,repeat the steps to configure a remote target ARN and add replication rules to the source bucket's replication config.
+Replication from a source bucket to multiple destination buckets is supported. For each of the targets, repeat the steps to configure a remote target ARN and add replication rules to the source bucket's replication config.
 
 Note that on the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` after replication succeeds to each of the targets. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle.
 
diff --git a/docs/bucket/replication/setup_3site_replication.sh b/docs/bucket/replication/setup_3site_replication.sh
new file mode 100755
index 000000000..ada4cf326
--- /dev/null
+++ b/docs/bucket/replication/setup_3site_replication.sh
@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+
+trap 'catch' ERR
+
+catch() {
+    echo "Cleaning up instances of MinIO"
+    pkill minio
+    rm -rf /tmp/multisitea
+    rm -rf /tmp/multisiteb
+    rm -rf /tmp/multisitec
+}
+
+catch
+
+set -e
+go install -v
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+export MINIO_PROMETHEUS_AUTH_TYPE=public
+
+minio server --address :9001 "http://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+      "http://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
+minio server --address :9002 "http://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+      "http://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
+
+minio server --address :9003 "http://localhost:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+      "http://localhost:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
+minio server --address :9004 "http://localhost:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+      "http://localhost:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
+
+minio server --address :9005 "http://localhost:9005/tmp/multisitec/data/disterasure/xl{1...4}" \
+      "http://localhost:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 &
+minio server --address :9006 "http://localhost:9005/tmp/multisitec/data/disterasure/xl{1...4}" \
+      "http://localhost:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 &
+
+sleep 30
+
+mc alias set sitea http://localhost:9001 minio minio123
+mc mb sitea/bucket
+mc version enable sitea/bucket
+mc mb -l sitea/olockbucket
+
+mc alias set siteb http://localhost:9004 minio minio123
+mc mb siteb/bucket/
+mc version enable siteb/bucket/
+mc mb -l siteb/olockbucket/
+
+mc alias set sitec http://localhost:9006 minio minio123
+mc mb sitec/bucket/
+mc version enable sitec/bucket/
+mc mb -l sitec/olockbucket
+
+echo "adding replication config for site a -> site b"
+remote_arn=$(mc admin bucket remote add sitea/bucket/ \
+   http://minio:minio123@localhost:9004/bucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+echo "adding replication rule for a -> b : ${remote_arn}"
+sleep 1
+mc replicate add sitea/bucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
+sleep 1
+
+echo "adding replication config for site b -> site a"
+remote_arn=$(mc admin bucket remote add siteb/bucket/ \
+   http://minio:minio123@localhost:9001/bucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for b -> a : ${remote_arn}"
+mc replicate add siteb/bucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
+sleep 1
+
+echo "adding replication config for site a -> site c"
+remote_arn=$(mc admin bucket remote add sitea/bucket/ \
+   http://minio:minio123@localhost:9006/bucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for a -> c : ${remote_arn}"
+mc replicate add sitea/bucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
+sleep 1
+echo "adding replication config for site c -> site a"
+remote_arn=$(mc admin bucket remote add sitec/bucket/ \
+   http://minio:minio123@localhost:9001/bucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for c -> a : ${remote_arn}"
+mc replicate add sitec/bucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
+sleep 1
+echo "adding replication config for site b -> site c"
+remote_arn=$(mc admin bucket remote add siteb/bucket/ \
+   http://minio:minio123@localhost:9006/bucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for b -> c : ${remote_arn}"
+mc replicate add siteb/bucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
+sleep 1
+
+echo "adding replication config for site c -> site b"
+remote_arn=$(mc admin bucket remote add sitec/bucket \
+   http://minio:minio123@localhost:9004/bucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for c -> b : ${remote_arn}"
+mc replicate add sitec/bucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
+sleep 1
+echo "adding replication config for olockbucket site a -> site b"
+remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \
+   http://minio:minio123@localhost:9004/olockbucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for olockbucket a -> b : ${remote_arn}"
+mc replicate add sitea/olockbucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
+sleep 1
+echo "adding replication config for site b -> site a"
+remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \
+   http://minio:minio123@localhost:9001/olockbucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for olockbucket b -> a : ${remote_arn}"
+mc replicate add siteb/olockbucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
+sleep 1
+echo "adding replication config for olockbucket site a -> site c"
+remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \
+   http://minio:minio123@localhost:9006/olockbucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for olockbucket a -> c : ${remote_arn}"
+mc replicate add sitea/olockbucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
+sleep 1
+echo "adding replication config for site c -> site a"
+remote_arn=$(mc admin bucket remote add sitec/olockbucket/ \
+   http://minio:minio123@localhost:9001/olockbucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for olockbucket c -> a : ${remote_arn}"
+mc replicate add sitec/olockbucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
+sleep 1
+echo "adding replication config for site b -> site c"
+remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \
+   http://minio:minio123@localhost:9006/olockbucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for olockbucket b -> c : ${remote_arn}"
+mc replicate add siteb/olockbucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
+sleep 1
+echo "adding replication config for site c -> site b"
+remote_arn=$(mc admin bucket remote add sitec/olockbucket \
+   http://minio:minio123@localhost:9004/olockbucket \
+   --service "replication" --json | jq -r ".RemoteARN")
+sleep 1
+echo "adding replication rule for olockbucket c -> b : ${remote_arn}"
+mc replicate add sitec/olockbucket/ \
+   --remote-bucket "${remote_arn}" \
+   --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
+sleep 1
+
+
+echo "Copying data to source sitea"
+mc cp --quiet /etc/hosts sitea/bucket
+sleep 1
+
+echo "Verifying the metadata difference between source and target"
+diff -pruN <(mc stat --json sitea/bucket/hosts | jq .) <(mc stat --json siteb/bucket/hosts | jq .)
+diff -pruN <(mc stat --json sitea/bucket/hosts | jq .) <(mc stat --json sitec/bucket/hosts | jq .)
diff --git a/docs/bucket/replication/setup_replication.sh b/docs/bucket/replication/setup_replication.sh
index 1d4529280..518a29891 100755
--- a/docs/bucket/replication/setup_replication.sh
+++ b/docs/bucket/replication/setup_replication.sh
@@ -1,10 +1,8 @@
 #!/bin/sh
 
-# create buckets with versioning enabled
-mc mb source/bucket --l    # --l flag enables object locking.If not needed, uncomment the 'mc version enable' commands
-mc mb dest/bucket  --l 
-#mc version enable source/bucket
-#mc version enable dest/bucket
+# Create buckets with versioning and object locking enabled.
+mc mb -l source/bucket
+mc mb -l dest/bucket
 
 #### Create a replication admin on source alias
 # create a replication admin user : repladmin
@@ -95,12 +93,13 @@ EOF
 mc admin policy add dest replpolicy ./replpolicy.json
 cat ./replpolicy.json
 
-#assign this replication policy to repluser
+# assign this replication policy to repluser
 mc admin policy set dest replpolicy user=repluser
 
 # define remote target for replication from source/bucket -> dest/bucket
-mc admin bucket remote add repladminAlias/bucket http://repluser:repluser123@localhost:9000/bucket --service replication --region us-east-1
+remote_arn=$(mc admin bucket remote add repladminAlias/bucket http://repluser:repluser123@localhost:9000/bucket --service replication --json | jq -r ".RemoteARN")
 
 echo "Now, use this ARN to add replication rules using 'mc replicate add' command"
 # use arn returned by above command to create a replication policy on the source/bucket with `mc replicate add`
-#mc replicate add source/bucket --priority 1 --remote-bucket bucket --arn arn:minio:replication:us-east-1:21fb52f5857473e2dbdcf62dcac21240861caf0f8301c3d26aad4de7677869c7:bucket --replicate delete-marker,delete
\ No newline at end of file
+mc replicate add source/bucket --priority 1 --remote-bucket "${remote_arn}" \
+   --replicate existing-objects,delete,delete-marker,replica-metadata-sync