add multi-site replication tests (#13631)

.github/workflows/iam-integrations.yaml

@@ -7,7 +7,7 @@ on:
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 

.github/workflows/replication.yaml

@@ -0,0 +1,33 @@
+name: Multi-site replication tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  replication-test:
+    name: Replication Tests with Go ${{ matrix.go-version }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Test Replication
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          go install -v github.com/minio/mc@latest
+          make test-replication

Makefile

@@ -42,16 +42,20 @@ test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
-test-race: verifiers build
+test-race: verifiers build ## builds minio, runs linters, tests (race)
 	@echo "Running unit tests under -race"
 	@(env bash $(PWD)/buildscripts/race.sh)
 
-test-iam: build
+test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends)"
 	@CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
 	@CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
+test-replication: install ## verify multi site replication
+	@echo "Running tests for Replication three sites"
+	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
+
 verify: ## verify minio various setups
 	@echo "Verifying build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

docs/bucket/replication/setup_3site_replication.sh

@@ -1,10 +1,22 @@
 #!/usr/bin/env bash
 
-trap 'catch' ERR
+trap 'catch $LINENO' ERR
 
+# shellcheck disable=SC2120
 catch() {
+    if [ $# -ne 0 ]; then
+       echo "error on line $1"
+       for site in sitea siteb sitec; do
+           echo "$site server logs ========="
+           cat "/tmp/${site}_1.log"
+           echo "==========================="
+           cat "/tmp/${site}_2.log"
+       done
+    fi
+
     echo "Cleaning up instances of MinIO"
     pkill minio
+    pkill -9 minio
     rm -rf /tmp/multisitea
     rm -rf /tmp/multisiteb
     rm -rf /tmp/multisitec
@@ -13,47 +25,47 @@ catch() {
 catch
 
 set -e
-go install -v
 export MINIO_BROWSER=off
 export MINIO_ROOT_USER="minio"
 export MINIO_ROOT_PASSWORD="minio123"
 export MINIO_PROMETHEUS_AUTH_TYPE=public
+export PATH=${GOPATH}/bin:${PATH}
 
-minio server --address :9001 "http://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
-      "http://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
+      "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
-minio server --address :9002 "http://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
-      "http://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
+      "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
 
-minio server --address :9003 "http://localhost:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
-      "http://localhost:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
+      "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
-minio server --address :9004 "http://localhost:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
-      "http://localhost:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
+      "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
 
-minio server --address :9005 "http://localhost:9005/tmp/multisitec/data/disterasure/xl{1...4}" \
+minio server --address 127.0.0.1:9005 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \
-      "http://localhost:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 &
+      "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 &
-minio server --address :9006 "http://localhost:9005/tmp/multisitec/data/disterasure/xl{1...4}" \
+minio server --address 127.0.0.1:9006 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \
-      "http://localhost:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 &
+      "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 &
 
 sleep 30
 
-mc alias set sitea http://localhost:9001 minio minio123
+mc alias set sitea http://127.0.0.1:9001 minio minio123
 mc mb sitea/bucket
 mc version enable sitea/bucket
 mc mb -l sitea/olockbucket
 
-mc alias set siteb http://localhost:9004 minio minio123
+mc alias set siteb http://127.0.0.1:9004 minio minio123
 mc mb siteb/bucket/
 mc version enable siteb/bucket/
 mc mb -l siteb/olockbucket/
 
-mc alias set sitec http://localhost:9006 minio minio123
+mc alias set sitec http://127.0.0.1:9006 minio minio123
 mc mb sitec/bucket/
 mc version enable sitec/bucket/
 mc mb -l sitec/olockbucket
 
 echo "adding replication config for site a -> site b"
 remote_arn=$(mc admin bucket remote add sitea/bucket/ \
-   http://minio:minio123@localhost:9004/bucket \
+   http://minio:minio123@127.0.0.1:9004/bucket \
    --service "replication" --json | jq -r ".RemoteARN")
 echo "adding replication rule for a -> b : ${remote_arn}"
 sleep 1
@@ -64,7 +76,7 @@ sleep 1
 
 echo "adding replication config for site b -> site a"
 remote_arn=$(mc admin bucket remote add siteb/bucket/ \
-   http://minio:minio123@localhost:9001/bucket \
+   http://minio:minio123@127.0.0.1:9001/bucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for b -> a : ${remote_arn}"
@@ -75,7 +87,7 @@ sleep 1
 
 echo "adding replication config for site a -> site c"
 remote_arn=$(mc admin bucket remote add sitea/bucket/ \
-   http://minio:minio123@localhost:9006/bucket \
+   http://minio:minio123@127.0.0.1:9006/bucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for a -> c : ${remote_arn}"
@@ -85,7 +97,7 @@ mc replicate add sitea/bucket/ \
 sleep 1
 echo "adding replication config for site c -> site a"
 remote_arn=$(mc admin bucket remote add sitec/bucket/ \
-   http://minio:minio123@localhost:9001/bucket \
+   http://minio:minio123@127.0.0.1:9001/bucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for c -> a : ${remote_arn}"
@@ -95,7 +107,7 @@ mc replicate add sitec/bucket/ \
 sleep 1
 echo "adding replication config for site b -> site c"
 remote_arn=$(mc admin bucket remote add siteb/bucket/ \
-   http://minio:minio123@localhost:9006/bucket \
+   http://minio:minio123@127.0.0.1:9006/bucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for b -> c : ${remote_arn}"
@@ -106,7 +118,7 @@ sleep 1
 
 echo "adding replication config for site c -> site b"
 remote_arn=$(mc admin bucket remote add sitec/bucket \
-   http://minio:minio123@localhost:9004/bucket \
+   http://minio:minio123@127.0.0.1:9004/bucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for c -> b : ${remote_arn}"
@@ -116,7 +128,7 @@ mc replicate add sitec/bucket/ \
 sleep 1
 echo "adding replication config for olockbucket site a -> site b"
 remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \
-   http://minio:minio123@localhost:9004/olockbucket \
+   http://minio:minio123@127.0.0.1:9004/olockbucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for olockbucket a -> b : ${remote_arn}"
@@ -126,7 +138,7 @@ mc replicate add sitea/olockbucket/ \
 sleep 1
 echo "adding replication config for site b -> site a"
 remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \
-   http://minio:minio123@localhost:9001/olockbucket \
+   http://minio:minio123@127.0.0.1:9001/olockbucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for olockbucket b -> a : ${remote_arn}"
@@ -136,7 +148,7 @@ mc replicate add siteb/olockbucket/ \
 sleep 1
 echo "adding replication config for olockbucket site a -> site c"
 remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \
-   http://minio:minio123@localhost:9006/olockbucket \
+   http://minio:minio123@127.0.0.1:9006/olockbucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for olockbucket a -> c : ${remote_arn}"
@@ -146,7 +158,7 @@ mc replicate add sitea/olockbucket/ \
 sleep 1
 echo "adding replication config for site c -> site a"
 remote_arn=$(mc admin bucket remote add sitec/olockbucket/ \
-   http://minio:minio123@localhost:9001/olockbucket \
+   http://minio:minio123@127.0.0.1:9001/olockbucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for olockbucket c -> a : ${remote_arn}"
@@ -156,7 +168,7 @@ mc replicate add sitec/olockbucket/ \
 sleep 1
 echo "adding replication config for site b -> site c"
 remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \
-   http://minio:minio123@localhost:9006/olockbucket \
+   http://minio:minio123@127.0.0.1:9006/olockbucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for olockbucket b -> c : ${remote_arn}"
@@ -166,7 +178,7 @@ mc replicate add siteb/olockbucket/ \
 sleep 1
 echo "adding replication config for site c -> site b"
 remote_arn=$(mc admin bucket remote add sitec/olockbucket \
-   http://minio:minio123@localhost:9004/olockbucket \
+   http://minio:minio123@127.0.0.1:9004/olockbucket \
    --service "replication" --json | jq -r ".RemoteARN")
 sleep 1
 echo "adding replication rule for olockbucket c -> b : ${remote_arn}"