diff --git a/.github/workflows/iam-integrations.yaml b/.github/workflows/iam-integrations.yaml index 141b22950..59dfd24fd 100644 --- a/.github/workflows/iam-integrations.yaml +++ b/.github/workflows/iam-integrations.yaml @@ -7,7 +7,7 @@ on: # This ensures that previous jobs for the PR are canceled when the PR is # updated. -concurrency: +concurrency: group: ${{ github.workflow }}-${{ github.head_ref }} cancel-in-progress: true diff --git a/.github/workflows/replication.yaml b/.github/workflows/replication.yaml new file mode 100644 index 000000000..4bd5d525b --- /dev/null +++ b/.github/workflows/replication.yaml @@ -0,0 +1,33 @@ +name: Multi-site replication tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +jobs: + replication-test: + name: Replication Tests with Go ${{ matrix.go-version }} + runs-on: ubuntu-latest + + strategy: + matrix: + go-version: [1.16.x, 1.17.x] + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Test Replication + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + go install -v github.com/minio/mc@latest + make test-replication diff --git a/Makefile b/Makefile index 9fd1ac309..9a4caf551 100644 --- a/Makefile +++ b/Makefile @@ -42,16 +42,20 @@ test: verifiers build ## builds minio, runs linters, tests @echo "Running unit tests" @GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null -test-race: verifiers build +test-race: verifiers build ## builds minio, runs linters, tests (race) @echo "Running unit tests under -race" @(env bash $(PWD)/buildscripts/race.sh) -test-iam: build +test-iam: build ## verify IAM (external IDP, etcd backends) @echo "Running tests for IAM (external IDP, etcd backends)" @CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd @echo "Running tests for IAM (external IDP, etcd backends) with -race" @CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd +test-replication: install ## verify multi site replication + @echo "Running tests for Replication three sites" + @(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh) + verify: ## verify minio various setups @echo "Verifying build with race" @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null diff --git a/docs/bucket/replication/setup_3site_replication.sh b/docs/bucket/replication/setup_3site_replication.sh index 1fc5180d6..7c07b326e 100755 --- a/docs/bucket/replication/setup_3site_replication.sh +++ b/docs/bucket/replication/setup_3site_replication.sh @@ -1,10 +1,22 @@ #!/usr/bin/env bash -trap 'catch' ERR +trap 'catch $LINENO' ERR +# shellcheck disable=SC2120 catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + for site in sitea siteb sitec; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + fi + echo "Cleaning up instances of MinIO" pkill minio + pkill -9 minio rm -rf /tmp/multisitea rm -rf /tmp/multisiteb rm -rf /tmp/multisitec @@ -13,47 +25,47 @@ catch() { catch set -e -go install -v export MINIO_BROWSER=off export MINIO_ROOT_USER="minio" export MINIO_ROOT_PASSWORD="minio123" export MINIO_PROMETHEUS_AUTH_TYPE=public +export PATH=${GOPATH}/bin:${PATH} -minio server --address :9001 "http://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & -minio server --address :9002 "http://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & +minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & -minio server --address :9003 "http://localhost:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://localhost:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & -minio server --address :9004 "http://localhost:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://localhost:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & +minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & -minio server --address :9005 "http://localhost:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ - "http://localhost:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 & -minio server --address :9006 "http://localhost:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ - "http://localhost:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 & +minio server --address 127.0.0.1:9005 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 & +minio server --address 127.0.0.1:9006 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 & sleep 30 -mc alias set sitea http://localhost:9001 minio minio123 +mc alias set sitea http://127.0.0.1:9001 minio minio123 mc mb sitea/bucket mc version enable sitea/bucket mc mb -l sitea/olockbucket -mc alias set siteb http://localhost:9004 minio minio123 +mc alias set siteb http://127.0.0.1:9004 minio minio123 mc mb siteb/bucket/ mc version enable siteb/bucket/ mc mb -l siteb/olockbucket/ -mc alias set sitec http://localhost:9006 minio minio123 +mc alias set sitec http://127.0.0.1:9006 minio minio123 mc mb sitec/bucket/ mc version enable sitec/bucket/ mc mb -l sitec/olockbucket echo "adding replication config for site a -> site b" remote_arn=$(mc admin bucket remote add sitea/bucket/ \ - http://minio:minio123@localhost:9004/bucket \ + http://minio:minio123@127.0.0.1:9004/bucket \ --service "replication" --json | jq -r ".RemoteARN") echo "adding replication rule for a -> b : ${remote_arn}" sleep 1 @@ -64,7 +76,7 @@ sleep 1 echo "adding replication config for site b -> site a" remote_arn=$(mc admin bucket remote add siteb/bucket/ \ - http://minio:minio123@localhost:9001/bucket \ + http://minio:minio123@127.0.0.1:9001/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for b -> a : ${remote_arn}" @@ -75,7 +87,7 @@ sleep 1 echo "adding replication config for site a -> site c" remote_arn=$(mc admin bucket remote add sitea/bucket/ \ - http://minio:minio123@localhost:9006/bucket \ + http://minio:minio123@127.0.0.1:9006/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for a -> c : ${remote_arn}" @@ -85,7 +97,7 @@ mc replicate add sitea/bucket/ \ sleep 1 echo "adding replication config for site c -> site a" remote_arn=$(mc admin bucket remote add sitec/bucket/ \ - http://minio:minio123@localhost:9001/bucket \ + http://minio:minio123@127.0.0.1:9001/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for c -> a : ${remote_arn}" @@ -95,7 +107,7 @@ mc replicate add sitec/bucket/ \ sleep 1 echo "adding replication config for site b -> site c" remote_arn=$(mc admin bucket remote add siteb/bucket/ \ - http://minio:minio123@localhost:9006/bucket \ + http://minio:minio123@127.0.0.1:9006/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for b -> c : ${remote_arn}" @@ -106,7 +118,7 @@ sleep 1 echo "adding replication config for site c -> site b" remote_arn=$(mc admin bucket remote add sitec/bucket \ - http://minio:minio123@localhost:9004/bucket \ + http://minio:minio123@127.0.0.1:9004/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for c -> b : ${remote_arn}" @@ -116,7 +128,7 @@ mc replicate add sitec/bucket/ \ sleep 1 echo "adding replication config for olockbucket site a -> site b" remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \ - http://minio:minio123@localhost:9004/olockbucket \ + http://minio:minio123@127.0.0.1:9004/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket a -> b : ${remote_arn}" @@ -126,7 +138,7 @@ mc replicate add sitea/olockbucket/ \ sleep 1 echo "adding replication config for site b -> site a" remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \ - http://minio:minio123@localhost:9001/olockbucket \ + http://minio:minio123@127.0.0.1:9001/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket b -> a : ${remote_arn}" @@ -136,7 +148,7 @@ mc replicate add siteb/olockbucket/ \ sleep 1 echo "adding replication config for olockbucket site a -> site c" remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \ - http://minio:minio123@localhost:9006/olockbucket \ + http://minio:minio123@127.0.0.1:9006/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket a -> c : ${remote_arn}" @@ -146,7 +158,7 @@ mc replicate add sitea/olockbucket/ \ sleep 1 echo "adding replication config for site c -> site a" remote_arn=$(mc admin bucket remote add sitec/olockbucket/ \ - http://minio:minio123@localhost:9001/olockbucket \ + http://minio:minio123@127.0.0.1:9001/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket c -> a : ${remote_arn}" @@ -156,7 +168,7 @@ mc replicate add sitec/olockbucket/ \ sleep 1 echo "adding replication config for site b -> site c" remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \ - http://minio:minio123@localhost:9006/olockbucket \ + http://minio:minio123@127.0.0.1:9006/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket b -> c : ${remote_arn}" @@ -166,7 +178,7 @@ mc replicate add siteb/olockbucket/ \ sleep 1 echo "adding replication config for site c -> site b" remote_arn=$(mc admin bucket remote add sitec/olockbucket \ - http://minio:minio123@localhost:9004/olockbucket \ + http://minio:minio123@127.0.0.1:9004/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket c -> b : ${remote_arn}"