diff --git a/.github/workflows/run-mint.sh b/.github/workflows/run-mint.sh index b6fd856e4..796a5aa54 100755 --- a/.github/workflows/run-mint.sh +++ b/.github/workflows/run-mint.sh @@ -19,26 +19,13 @@ docker-compose -f minio-${MODE}.yaml up -d sleep 5m docker run --rm --net=host \ - --name="mint-${MODE}-${JOB_NAME}" \ - -e SERVER_ENDPOINT="127.0.0.1:9000" \ - -e ACCESS_KEY="${ACCESS_KEY}" \ - -e SECRET_KEY="${SECRET_KEY}" \ - -e ENABLE_HTTPS=0 \ - -e MINT_MODE="${MINT_MODE}" \ - docker.io/minio/mint:edge \ - aws-sdk-go \ - aws-sdk-java \ - aws-sdk-php \ - aws-sdk-ruby \ - awscli \ - healthcheck \ - mc \ - minio-go \ - minio-java \ - minio-py \ - s3cmd \ - s3select \ - versioning + --name="mint-${MODE}-${JOB_NAME}" \ + -e SERVER_ENDPOINT="127.0.0.1:9000" \ + -e ACCESS_KEY="${ACCESS_KEY}" \ + -e SECRET_KEY="${SECRET_KEY}" \ + -e ENABLE_HTTPS=0 \ + -e MINT_MODE="${MINT_MODE}" \ + docker.io/minio/mint:edge docker-compose -f minio-${MODE}.yaml down || true sleep 10s @@ -49,4 +36,3 @@ docker volume rm $(docker volume ls -f dangling=true) || true ## change working directory cd ../../../ - diff --git a/.github/workflows/shfmt.yml b/.github/workflows/shfmt.yml new file mode 100644 index 000000000..903ca6701 --- /dev/null +++ b/.github/workflows/shfmt.yml @@ -0,0 +1,22 @@ +name: Shell formatting checks + +on: + pull_request: + branches: + - master + +permissions: + contents: read + +jobs: + build: + name: runner / shfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: luizm/action-sh-checker@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SHFMT_OPTS: "-s" + with: + sh_checker_shellcheck_disable: true # disable for now diff --git a/buildscripts/checkdeps.sh b/buildscripts/checkdeps.sh index 63d550fa4..11ecc4db0 100644 --- a/buildscripts/checkdeps.sh +++ b/buildscripts/checkdeps.sh @@ -3,19 +3,19 @@ _init() { - shopt -s extglob + shopt -s extglob - ## Minimum required versions for build dependencies - GIT_VERSION="1.0" - GO_VERSION="1.16" - OSX_VERSION="10.8" - KNAME=$(uname -s) - ARCH=$(uname -m) - case "${KNAME}" in - SunOS ) - ARCH=$(isainfo -k) - ;; - esac + ## Minimum required versions for build dependencies + GIT_VERSION="1.0" + GO_VERSION="1.16" + OSX_VERSION="10.8" + KNAME=$(uname -s) + ARCH=$(uname -m) + case "${KNAME}" in + SunOS) + ARCH=$(isainfo -k) + ;; + esac } ## FIXME: @@ -28,24 +28,23 @@ _init() { ## } ## readlink() { - TARGET_FILE=$1 + TARGET_FILE=$1 - cd `dirname $TARGET_FILE` - TARGET_FILE=`basename $TARGET_FILE` + cd $(dirname $TARGET_FILE) + TARGET_FILE=$(basename $TARGET_FILE) - # Iterate down a (possible) chain of symlinks - while [ -L "$TARGET_FILE" ] - do - TARGET_FILE=$(env readlink $TARGET_FILE) - cd `dirname $TARGET_FILE` - TARGET_FILE=`basename $TARGET_FILE` - done + # Iterate down a (possible) chain of symlinks + while [ -L "$TARGET_FILE" ]; do + TARGET_FILE=$(env readlink $TARGET_FILE) + cd $(dirname $TARGET_FILE) + TARGET_FILE=$(basename $TARGET_FILE) + done - # Compute the canonicalized name by finding the physical path - # for the directory we're in and appending the target file. - PHYS_DIR=`pwd -P` - RESULT=$PHYS_DIR/$TARGET_FILE - echo $RESULT + # Compute the canonicalized name by finding the physical path + # for the directory we're in and appending the target file. + PHYS_DIR=$(pwd -P) + RESULT=$PHYS_DIR/$TARGET_FILE + echo $RESULT } ## FIXME: @@ -59,84 +58,86 @@ readlink() { ## } ## check_minimum_version() { - IFS='.' read -r -a varray1 <<< "$1" - IFS='.' read -r -a varray2 <<< "$2" + IFS='.' read -r -a varray1 <<<"$1" + IFS='.' read -r -a varray2 <<<"$2" - for i in "${!varray1[@]}"; do - if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then - return 0 - elif [[ ${varray1[i]} -gt ${varray2[i]} ]]; then - return 1 - fi - done + for i in "${!varray1[@]}"; do + if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then + return 0 + elif [[ ${varray1[i]} -gt ${varray2[i]} ]]; then + return 1 + fi + done - return 0 + return 0 } assert_is_supported_arch() { - case "${ARCH}" in - x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64 ) - return - ;; - *) - echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64]" - exit 1 - esac + case "${ARCH}" in + x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64) + return + ;; + *) + echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64]" + exit 1 + ;; + esac } assert_is_supported_os() { - case "${KNAME}" in - Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS ) - return - ;; - Darwin ) - osx_host_version=$(env sw_vers -productVersion) - if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then - echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}" - exit 1 - fi - return - ;; - *) - echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]" - exit 1 - esac + case "${KNAME}" in + Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS) + return + ;; + Darwin) + osx_host_version=$(env sw_vers -productVersion) + if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then + echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}" + exit 1 + fi + return + ;; + *) + echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]" + exit 1 + ;; + esac } assert_check_golang_env() { - if ! which go >/dev/null 2>&1; then - echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://golang.org/doc/install" - exit 1 - fi + if ! which go >/dev/null 2>&1; then + echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://golang.org/doc/install" + exit 1 + fi - installed_go_version=$(go version | sed 's/^.* go\([0-9.]*\).*$/\1/') - if ! check_minimum_version "${GO_VERSION}" "${installed_go_version}"; then - echo "Go runtime version '${installed_go_version}' is unsupported. Minimum supported version: ${GO_VERSION} to compile." - exit 1 - fi + installed_go_version=$(go version | sed 's/^.* go\([0-9.]*\).*$/\1/') + if ! check_minimum_version "${GO_VERSION}" "${installed_go_version}"; then + echo "Go runtime version '${installed_go_version}' is unsupported. Minimum supported version: ${GO_VERSION} to compile." + exit 1 + fi } assert_check_deps() { - # support unusual Git versions such as: 2.7.4 (Apple Git-66) - installed_git_version=$(git version | perl -ne '$_ =~ m/git version (.*?)( |$)/; print "$1\n";') - if ! check_minimum_version "${GIT_VERSION}" "${installed_git_version}"; then - echo "Git version '${installed_git_version}' is not supported. Minimum supported version: ${GIT_VERSION}" - exit 1 - fi + # support unusual Git versions such as: 2.7.4 (Apple Git-66) + installed_git_version=$(git version | perl -ne '$_ =~ m/git version (.*?)( |$)/; print "$1\n";') + if ! check_minimum_version "${GIT_VERSION}" "${installed_git_version}"; then + echo "Git version '${installed_git_version}' is not supported. Minimum supported version: ${GIT_VERSION}" + exit 1 + fi } main() { - ## Check for supported arch - assert_is_supported_arch + ## Check for supported arch + assert_is_supported_arch - ## Check for supported os - assert_is_supported_os + ## Check for supported os + assert_is_supported_os - ## Check for Go environment - assert_check_golang_env + ## Check for Go environment + assert_check_golang_env - ## Check for dependencies - assert_check_deps + ## Check for dependencies + assert_check_deps } _init && main "$@" diff --git a/buildscripts/cross-compile.sh b/buildscripts/cross-compile.sh index a920edb6b..691891bae 100755 --- a/buildscripts/cross-compile.sh +++ b/buildscripts/cross-compile.sh @@ -5,33 +5,33 @@ set -e [ -n "$BASH_XTRACEFD" ] && set -x function _init() { - ## All binaries are static make sure to disable CGO. - export CGO_ENABLED=0 + ## All binaries are static make sure to disable CGO. + export CGO_ENABLED=0 - ## List of architectures and OS to test coss compilation. - SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64" + ## List of architectures and OS to test coss compilation. + SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64" } function _build() { - local osarch=$1 - IFS=/ read -r -a arr <<<"$osarch" - os="${arr[0]}" - arch="${arr[1]}" - package=$(go list -f '{{.ImportPath}}') - printf -- "--> %15s:%s\n" "${osarch}" "${package}" + local osarch=$1 + IFS=/ read -r -a arr <<<"$osarch" + os="${arr[0]}" + arch="${arr[1]}" + package=$(go list -f '{{.ImportPath}}') + printf -- "--> %15s:%s\n" "${osarch}" "${package}" - # go build -trimpath to build the binary. - export GOOS=$os - export GOARCH=$arch - export GO111MODULE=on - go build -trimpath -tags kqueue -o /dev/null + # go build -trimpath to build the binary. + export GOOS=$os + export GOARCH=$arch + export GO111MODULE=on + go build -trimpath -tags kqueue -o /dev/null } function main() { - echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}" - for each_osarch in ${SUPPORTED_OSARCH}; do - _build "${each_osarch}" - done + echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}" + for each_osarch in ${SUPPORTED_OSARCH}; do + _build "${each_osarch}" + done } _init && main "$@" diff --git a/buildscripts/disable-root.sh b/buildscripts/disable-root.sh index 7f9068eca..7234a7043 100755 --- a/buildscripts/disable-root.sh +++ b/buildscripts/disable-root.sh @@ -12,21 +12,21 @@ nr_servers=4 addr="localhost" args="" -for ((i=0;i<$[${nr_servers}];i++)); do - args="$args $scheme://$addr:$[9100+$i]/${HOME}/tmp/dist/path1/$i" +for ((i = 0; i < $((nr_servers)); i++)); do + args="$args $scheme://$addr:$((9100 + i))/${HOME}/tmp/dist/path1/$i" done echo $args -for ((i=0;i<$[${nr_servers}];i++)); do - (minio server --address ":$[9100+$i]" $args 2>&1 > /tmp/log$i.txt) & +for ((i = 0; i < $((nr_servers)); i++)); do + (minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) & done sleep 10s if [ ! -f ./mc ]; then - wget --quiet -O ./mc https://dl.minio.io/client/mc/release/linux-amd64/./mc && \ - chmod +x mc + wget --quiet -O ./mc https://dl.minio.io/client/mc/release/linux-amd64/./mc && + chmod +x mc fi set +e @@ -41,8 +41,8 @@ sleep 3s # let things settle a little ./mc ls minioadm/ if [ $? -eq 0 ]; then - echo "listing succeeded, 'minioadmin' was not disabled" - exit 1 + echo "listing succeeded, 'minioadmin' was not disabled" + exit 1 fi set -e @@ -50,16 +50,16 @@ set -e killall -9 minio export MINIO_API_ROOT_ACCESS=on -for ((i=0;i<$[${nr_servers}];i++)); do - (minio server --address ":$[9100+$i]" $args 2>&1 > /tmp/log$i.txt) & +for ((i = 0; i < $((nr_servers)); i++)); do + (minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) & done set +e ./mc ls minioadm/ if [ $? -ne 0 ]; then - echo "listing failed, 'minioadmin' should be enabled" - exit 1 + echo "listing failed, 'minioadmin' should be enabled" + exit 1 fi killall -9 minio @@ -70,14 +70,14 @@ rm -rf /tmp/multisiteb/ echo "Setup site-replication and then disable root credentials" minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & sleep 20s @@ -98,14 +98,14 @@ echo "turning off root access, however site replication must continue" export MINIO_API_ROOT_ACCESS=off minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & sleep 20s diff --git a/buildscripts/heal-inconsistent-versions.sh b/buildscripts/heal-inconsistent-versions.sh index f88e54712..f4c918a65 100755 --- a/buildscripts/heal-inconsistent-versions.sh +++ b/buildscripts/heal-inconsistent-versions.sh @@ -6,88 +6,87 @@ set -x WORK_DIR="$PWD/.verify-$RANDOM" MINIO_CONFIG_DIR="$WORK_DIR/.minio" -MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server ) +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) if [ ! -x "$PWD/minio" ]; then - echo "minio executable binary not found in current directory" - exit 1 + echo "minio executable binary not found in current directory" + exit 1 fi if [ ! -x "$PWD/minio" ]; then - echo "minio executable binary not found in current directory" - exit 1 + echo "minio executable binary not found in current directory" + exit 1 fi function start_minio_4drive() { - start_port=$1 + start_port=$1 - export MINIO_ROOT_USER=minio - export MINIO_ROOT_PASSWORD=minio123 - export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" - unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects - export MINIO_CI_CD=1 + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export MINIO_CI_CD=1 - mkdir ${WORK_DIR} - C_PWD=${PWD} - if [ ! -x "$PWD/mc" ]; then - MC_BUILD_DIR="mc-$RANDOM" - if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then - echo "failed to download https://github.com/minio/mc" - purge "${MC_BUILD_DIR}" - exit 1 + mkdir ${WORK_DIR} + C_PWD=${PWD} + if [ ! -x "$PWD/mc" ]; then + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc") + + # remove mc source. + purge "${MC_BUILD_DIR}" fi - (cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc") + "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 5 - # remove mc source. - purge "${MC_BUILD_DIR}" - fi + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" > "${WORK_DIR}/server1.log" 2>&1 & - pid=$! - disown $pid - sleep 5 + "${PWD}/mc" mb --with-versioning minio/bucket - if ! ps -p ${pid} 1>&2 >/dev/null; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + for i in $(seq 1 4); do + "${PWD}/mc" cp /etc/hosts minio/bucket/testobj - "${PWD}/mc" mb --with-versioning minio/bucket + sudo chown -R root. "${WORK_DIR}/disk${i}" - for i in $(seq 1 4); do - "${PWD}/mc" cp /etc/hosts minio/bucket/testobj + "${PWD}/mc" cp /etc/hosts minio/bucket/testobj - sudo chown -R root. "${WORK_DIR}/disk${i}" + sudo chown -R ${USER}. "${WORK_DIR}/disk${i}" + done - "${PWD}/mc" cp /etc/hosts minio/bucket/testobj + for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do + "${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum + done - sudo chown -R ${USER}. "${WORK_DIR}/disk${i}" - done - - for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do - "${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum - done - - pkill minio - sleep 3 + pkill minio + sleep 3 } function main() { - start_port=$(shuf -i 10000-65000 -n 1) + start_port=$(shuf -i 10000-65000 -n 1) - start_minio_4drive ${start_port} + start_minio_4drive ${start_port} } -function purge() -{ - rm -rf "$1" +function purge() { + rm -rf "$1" } -( main "$@" ) +(main "$@") rv=$? purge "$WORK_DIR" exit "$rv" diff --git a/buildscripts/minio-upgrade.sh b/buildscripts/minio-upgrade.sh index 745f513c1..5721a8c64 100644 --- a/buildscripts/minio-upgrade.sh +++ b/buildscripts/minio-upgrade.sh @@ -4,89 +4,89 @@ trap 'cleanup $LINENO' ERR # shellcheck disable=SC2120 cleanup() { - MINIO_VERSION=dev docker-compose \ - -f "buildscripts/upgrade-tests/compose.yml" \ - rm -s -f - docker volume prune -f + MINIO_VERSION=dev docker-compose \ + -f "buildscripts/upgrade-tests/compose.yml" \ + rm -s -f + docker volume prune -f } verify_checksum_after_heal() { - local sum1 - sum1=$(curl -s "$2" | sha256sum); - mc admin heal --json -r "$1" >/dev/null; # test after healing - local sum1_heal - sum1_heal=$(curl -s "$2" | sha256sum); + local sum1 + sum1=$(curl -s "$2" | sha256sum) + mc admin heal --json -r "$1" >/dev/null # test after healing + local sum1_heal + sum1_heal=$(curl -s "$2" | sha256sum) - if [ "${sum1_heal}" != "${sum1}" ]; then - echo "mismatch expected ${sum1_heal}, got ${sum1}" - exit 1; - fi + if [ "${sum1_heal}" != "${sum1}" ]; then + echo "mismatch expected ${sum1_heal}, got ${sum1}" + exit 1 + fi } verify_checksum_mc() { - local expected - expected=$(mc cat "$1" | sha256sum) - local got - got=$(mc cat "$2" | sha256sum) + local expected + expected=$(mc cat "$1" | sha256sum) + local got + got=$(mc cat "$2" | sha256sum) - if [ "${expected}" != "${got}" ]; then - echo "mismatch - expected ${expected}, got ${got}" - exit 1; - fi - echo "matches - ${expected}, got ${got}" + if [ "${expected}" != "${got}" ]; then + echo "mismatch - expected ${expected}, got ${got}" + exit 1 + fi + echo "matches - ${expected}, got ${got}" } add_alias() { - for i in $(seq 1 4); do - echo "... attempting to add alias $i" - until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do - echo "...waiting... for 5secs" && sleep 5 - done - done + for i in $(seq 1 4); do + echo "... attempting to add alias $i" + until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do + echo "...waiting... for 5secs" && sleep 5 + done + done - echo "Sleeping for nginx" - sleep 20 + echo "Sleeping for nginx" + sleep 20 } __init__() { - sudo apt install curl -y - export GOPATH=/tmp/gopath - export PATH=${PATH}:${GOPATH}/bin + sudo apt install curl -y + export GOPATH=/tmp/gopath + export PATH=${PATH}:${GOPATH}/bin - go install github.com/minio/mc@latest + go install github.com/minio/mc@latest - TAG=minio/minio:dev make docker + TAG=minio/minio:dev make docker - MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \ - -f "buildscripts/upgrade-tests/compose.yml" \ - up -d --build + MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \ + -f "buildscripts/upgrade-tests/compose.yml" \ + up -d --build - add_alias + add_alias - mc mb minio/minio-test/ - mc cp ./minio minio/minio-test/to-read/ - mc cp /etc/hosts minio/minio-test/to-read/hosts - mc anonymous set download minio/minio-test + mc mb minio/minio-test/ + mc cp ./minio minio/minio-test/to-read/ + mc cp /etc/hosts minio/minio-test/to-read/hosts + mc anonymous set download minio/minio-test - verify_checksum_mc ./minio minio/minio-test/to-read/minio + verify_checksum_mc ./minio minio/minio-test/to-read/minio - curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum + curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum - MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop + MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop } main() { - MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build + MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build - add_alias + add_alias - verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts + verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts - verify_checksum_mc ./minio minio/minio-test/to-read/minio + verify_checksum_mc ./minio minio/minio-test/to-read/minio - verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts + verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts - cleanup + cleanup } -( __init__ "$@" && main "$@" ) +(__init__ "$@" && main "$@") diff --git a/buildscripts/race.sh b/buildscripts/race.sh index 2aa24f9c6..2931342dc 100755 --- a/buildscripts/race.sh +++ b/buildscripts/race.sh @@ -6,5 +6,5 @@ export GORACE="history_size=7" export MINIO_API_REQUESTS_MAX=10000 for d in $(go list ./...); do - CGO_ENABLED=1 go test -v -race --timeout 100m "$d" + CGO_ENABLED=1 go test -v -race --timeout 100m "$d" done diff --git a/buildscripts/resolve-right-versions.sh b/buildscripts/resolve-right-versions.sh index 61d2c149f..5e33b8300 100755 --- a/buildscripts/resolve-right-versions.sh +++ b/buildscripts/resolve-right-versions.sh @@ -6,67 +6,66 @@ set -x WORK_DIR="$PWD/.verify-$RANDOM" MINIO_CONFIG_DIR="$WORK_DIR/.minio" -MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server ) +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) if [ ! -x "$PWD/minio" ]; then - echo "minio executable binary not found in current directory" - exit 1 + echo "minio executable binary not found in current directory" + exit 1 fi function start_minio_5drive() { - start_port=$1 + start_port=$1 - export MINIO_ROOT_USER=minio - export MINIO_ROOT_PASSWORD=minio123 - export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" - unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects - export MINIO_CI_CD=1 + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export MINIO_CI_CD=1 - MC_BUILD_DIR="mc-$RANDOM" - if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then - echo "failed to download https://github.com/minio/mc" + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + + # remove mc source. purge "${MC_BUILD_DIR}" - exit 1 - fi - (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + "${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/" - # remove mc source. - purge "${MC_BUILD_DIR}" + "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 5 - "${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/" + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 & - pid=$! - disown $pid - sleep 5 + "${WORK_DIR}/mc" stat minio/bucket/testobj - if ! ps -p ${pid} 1>&2 >/dev/null; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi - - "${WORK_DIR}/mc" stat minio/bucket/testobj - - pkill minio - sleep 3 + pkill minio + sleep 3 } function main() { - start_port=$(shuf -i 10000-65000 -n 1) + start_port=$(shuf -i 10000-65000 -n 1) - start_minio_5drive ${start_port} + start_minio_5drive ${start_port} } -function purge() -{ - rm -rf "$1" +function purge() { + rm -rf "$1" } -( main "$@" ) +(main "$@") rv=$? purge "$WORK_DIR" exit "$rv" diff --git a/buildscripts/rewrite-old-new.sh b/buildscripts/rewrite-old-new.sh index 6cc015519..55c4436c9 100755 --- a/buildscripts/rewrite-old-new.sh +++ b/buildscripts/rewrite-old-new.sh @@ -6,146 +6,151 @@ set -x WORK_DIR="$PWD/.verify-$RANDOM" MINIO_CONFIG_DIR="$WORK_DIR/.minio" -MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server ) -MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server ) +MINIO_OLD=("$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server) +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) if [ ! -x "$PWD/minio" ]; then - echo "minio executable binary not found in current directory" - exit 1 + echo "minio executable binary not found in current directory" + exit 1 fi function download_old_release() { - if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then - curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z - chmod a+x minio.RELEASE.2020-10-28T08-16-50Z - fi + if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then + curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z + chmod a+x minio.RELEASE.2020-10-28T08-16-50Z + fi } function verify_rewrite() { - start_port=$1 + start_port=$1 - export MINIO_ACCESS_KEY=minio - export MINIO_SECRET_KEY=minio123 - export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" - unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects - export MINIO_CI_CD=1 + export MINIO_ACCESS_KEY=minio + export MINIO_SECRET_KEY=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export MINIO_CI_CD=1 - MC_BUILD_DIR="mc-$RANDOM" - if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then - echo "failed to download https://github.com/minio/mc" + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + + # remove mc source. purge "${MC_BUILD_DIR}" - exit 1 - fi - (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 10 - # remove mc source. - purge "${MC_BUILD_DIR}" + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 & - pid=$! - disown $pid - sleep 10 + "${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock + "${WORK_DIR}/mc" cp \ + buildscripts/verify-build.sh \ + minio/healing-rewrite-bucket/ \ + --disable-multipart --quiet - if ! ps -p ${pid} 1>&2 >/dev/null; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + "${WORK_DIR}/mc" cp \ + buildscripts/verify-build.sh \ + minio/healing-rewrite-bucket/ \ + --disable-multipart --quiet - "${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock - "${WORK_DIR}/mc" cp \ - buildscripts/verify-build.sh \ - minio/healing-rewrite-bucket/ \ - --disable-multipart --quiet + "${WORK_DIR}/mc" cp \ + buildscripts/verify-build.sh \ + minio/healing-rewrite-bucket/ \ + --disable-multipart --quiet - "${WORK_DIR}/mc" cp \ - buildscripts/verify-build.sh \ - minio/healing-rewrite-bucket/ \ - --disable-multipart --quiet + kill ${pid} + sleep 3 - "${WORK_DIR}/mc" cp \ - buildscripts/verify-build.sh \ - minio/healing-rewrite-bucket/ \ - --disable-multipart --quiet + "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 10 - kill ${pid} - sleep 3 + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 & - pid=$! - disown $pid - sleep 10 + go build ./docs/debugging/s3-check-md5/ + if ! ./s3-check-md5 \ + -debug \ + -versions \ + -access-key minio \ + -secret-key minio123 \ + -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + mkdir -p inspects + ( + cd inspects + "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/** + ) - if ! ps -p ${pid} 1>&2 >/dev/null; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + "${WORK_DIR}/mc" mb play/inspects + "${WORK_DIR}/mc" mirror inspects play/inspects - go build ./docs/debugging/s3-check-md5/ - if ! ./s3-check-md5 \ - -debug \ - -versions \ - -access-key minio \ - -secret-key minio123 \ - -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - mkdir -p inspects - (cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**) + purge "$WORK_DIR" + exit 1 + fi - "${WORK_DIR}/mc" mb play/inspects - "${WORK_DIR}/mc" mirror inspects play/inspects + go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123" + sleep 1 - purge "$WORK_DIR" - exit 1 - fi + if ! ./s3-check-md5 \ + -debug \ + -versions \ + -access-key minio \ + -secret-key minio123 \ + -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + mkdir -p inspects + ( + cd inspects + "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/** + ) - go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123" - sleep 1 + "${WORK_DIR}/mc" mb play/inspects + "${WORK_DIR}/mc" mirror inspects play/inspects - if ! ./s3-check-md5 \ - -debug \ - -versions \ - -access-key minio \ - -secret-key minio123 \ - -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - mkdir -p inspects - (cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**) + purge "$WORK_DIR" + exit 1 + fi - "${WORK_DIR}/mc" mb play/inspects - "${WORK_DIR}/mc" mirror inspects play/inspects - - purge "$WORK_DIR" - exit 1 - fi - - kill ${pid} + kill ${pid} } function main() { - download_old_release + download_old_release - start_port=$(shuf -i 10000-65000 -n 1) + start_port=$(shuf -i 10000-65000 -n 1) - verify_rewrite ${start_port} + verify_rewrite ${start_port} } -function purge() -{ - rm -rf "$1" +function purge() { + rm -rf "$1" } -( main "$@" ) +(main "$@") rv=$? purge "$WORK_DIR" exit "$rv" diff --git a/buildscripts/unaligned-healing.sh b/buildscripts/unaligned-healing.sh index fc2e5ea4d..d605eba92 100755 --- a/buildscripts/unaligned-healing.sh +++ b/buildscripts/unaligned-healing.sh @@ -6,167 +6,172 @@ set -o pipefail set -x if [ ! -x "$PWD/minio" ]; then - echo "minio executable binary not found in current directory" - exit 1 + echo "minio executable binary not found in current directory" + exit 1 fi WORK_DIR="$PWD/.verify-$RANDOM" MINIO_CONFIG_DIR="$WORK_DIR/.minio" -MINIO_OLD=( "$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server ) -MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server ) +MINIO_OLD=("$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server) +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) function download_old_release() { - if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then - curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z - chmod a+x minio.RELEASE.2021-11-24T23-19-33Z - fi + if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then + curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z + chmod a+x minio.RELEASE.2021-11-24T23-19-33Z + fi } function start_minio_16drive() { - start_port=$1 + start_port=$1 - export MINIO_ROOT_USER=minio - export MINIO_ROOT_PASSWORD=minio123 - export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" - unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects - export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests - export MINIO_CI_CD=1 + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests + export MINIO_CI_CD=1 - MC_BUILD_DIR="mc-$RANDOM" - if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then - echo "failed to download https://github.com/minio/mc" + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + + # remove mc source. purge "${MC_BUILD_DIR}" - exit 1 - fi - (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 30 - # remove mc source. - purge "${MC_BUILD_DIR}" + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 & - pid=$! - disown $pid - sleep 30 + shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null + "${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet + "${WORK_DIR}/mc" cp \ + "${WORK_DIR}/unaligned" \ + minio/healing-shard-bucket/unaligned \ + --disable-multipart --quiet - if ! ps -p ${pid} 1>&2 >/dev/null; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + ## "unaligned" object name gets consistently distributed + ## to disks in following distribution order + ## + ## NOTE: if you change the name make sure to change the + ## distribution order present here + ## + ## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] - shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null - "${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet - "${WORK_DIR}/mc" cp \ - "${WORK_DIR}/unaligned" \ - minio/healing-shard-bucket/unaligned \ - --disable-multipart --quiet + ## make sure to remove the "last" data shard + rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned" + sleep 10 + ## Heal the shard + "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket + ## then remove any other data shard let's pick first disk + ## - 1st data shard. + rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned" + sleep 10 - ## "unaligned" object name gets consistently distributed - ## to disks in following distribution order - ## - ## NOTE: if you change the name make sure to change the - ## distribution order present here - ## - ## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + go build ./docs/debugging/s3-check-md5/ + if ! ./s3-check-md5 \ + -debug \ + -access-key minio \ + -secret-key minio123 \ + -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - ## make sure to remove the "last" data shard - rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned" - sleep 10 - ## Heal the shard - "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket - ## then remove any other data shard let's pick first disk - ## - 1st data shard. - rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned" - sleep 10 + pkill minio + sleep 3 - go build ./docs/debugging/s3-check-md5/ - if ! ./s3-check-md5 \ - -debug \ - -access-key minio \ - -secret-key minio123 \ - -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 30 - pkill minio - sleep 3 + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 & - pid=$! - disown $pid - sleep 30 + if ! ./s3-check-md5 \ + -debug \ + -access-key minio \ + -secret-key minio123 \ + -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + mkdir -p inspects + ( + cd inspects + "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/** + ) - if ! ps -p ${pid} 1>&2 >/dev/null; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + "${WORK_DIR}/mc" mb play/inspects + "${WORK_DIR}/mc" mirror inspects play/inspects - if ! ./s3-check-md5 \ - -debug \ - -access-key minio \ - -secret-key minio123 \ - -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - mkdir -p inspects - (cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**) + purge "$WORK_DIR" + exit 1 + fi - "${WORK_DIR}/mc" mb play/inspects - "${WORK_DIR}/mc" mirror inspects play/inspects + "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket - purge "$WORK_DIR" - exit 1 - fi + if ! ./s3-check-md5 \ + -debug \ + -access-key minio \ + -secret-key minio123 \ + -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + mkdir -p inspects + ( + cd inspects + "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/** + ) - "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket + "${WORK_DIR}/mc" mb play/inspects + "${WORK_DIR}/mc" mirror inspects play/inspects - if ! ./s3-check-md5 \ - -debug \ - -access-key minio \ - -secret-key minio123 \ - -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then - echo "server1 log:" - cat "${WORK_DIR}/server1.log" - echo "FAILED" - mkdir -p inspects - (cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**) + purge "$WORK_DIR" + exit 1 + fi - "${WORK_DIR}/mc" mb play/inspects - "${WORK_DIR}/mc" mirror inspects play/inspects - - purge "$WORK_DIR" - exit 1 - fi - - pkill minio - sleep 3 + pkill minio + sleep 3 } function main() { - download_old_release + download_old_release - start_port=$(shuf -i 10000-65000 -n 1) + start_port=$(shuf -i 10000-65000 -n 1) - start_minio_16drive ${start_port} + start_minio_16drive ${start_port} } -function purge() -{ - rm -rf "$1" +function purge() { + rm -rf "$1" } -( main "$@" ) +(main "$@") rv=$? purge "$WORK_DIR" exit "$rv" diff --git a/buildscripts/verify-build.sh b/buildscripts/verify-build.sh index ad76549b3..c3e4f3f83 100755 --- a/buildscripts/verify-build.sh +++ b/buildscripts/verify-build.sh @@ -6,8 +6,8 @@ set -E set -o pipefail if [ ! -x "$PWD/minio" ]; then - echo "minio executable binary not found in current directory" - exit 1 + echo "minio executable binary not found in current directory" + exit 1 fi WORK_DIR="$PWD/.verify-$RANDOM" @@ -25,285 +25,270 @@ export ENABLE_ADMIN=1 export MINIO_CI_CD=1 MINIO_CONFIG_DIR="$WORK_DIR/.minio" -MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" ) +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR") FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB" FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB" FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh" -function start_minio_fs() -{ - export MINIO_ROOT_USER=$ACCESS_KEY - export MINIO_ROOT_PASSWORD=$SECRET_KEY - "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 & - sleep 10 +function start_minio_fs() { + export MINIO_ROOT_USER=$ACCESS_KEY + export MINIO_ROOT_PASSWORD=$SECRET_KEY + "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 & + sleep 10 } -function start_minio_erasure() -{ - "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 & - sleep 15 +function start_minio_erasure() { + "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 & + sleep 15 } -function start_minio_erasure_sets() -{ - export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}" - "${MINIO[@]}" server > "$WORK_DIR/erasure-minio-sets.log" 2>&1 & - sleep 15 +function start_minio_erasure_sets() { + export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}" + "${MINIO[@]}" server >"$WORK_DIR/erasure-minio-sets.log" 2>&1 & + sleep 15 } -function start_minio_pool_erasure_sets() -{ - export MINIO_ROOT_USER=$ACCESS_KEY - export MINIO_ROOT_PASSWORD=$SECRET_KEY - export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}" - "${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 & - "${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 & +function start_minio_pool_erasure_sets() { + export MINIO_ROOT_USER=$ACCESS_KEY + export MINIO_ROOT_PASSWORD=$SECRET_KEY + export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}" + "${MINIO[@]}" server --address ":9000" >"$WORK_DIR/pool-minio-9000.log" 2>&1 & + "${MINIO[@]}" server --address ":9001" >"$WORK_DIR/pool-minio-9001.log" 2>&1 & - sleep 40 + sleep 40 } -function start_minio_pool_erasure_sets_ipv6() -{ - export MINIO_ROOT_USER=$ACCESS_KEY - export MINIO_ROOT_PASSWORD=$SECRET_KEY - export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}" - "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 & - "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 & +function start_minio_pool_erasure_sets_ipv6() { + export MINIO_ROOT_USER=$ACCESS_KEY + export MINIO_ROOT_PASSWORD=$SECRET_KEY + export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}" + "${MINIO[@]}" server --address="[::1]:9000" >"$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 & + "${MINIO[@]}" server --address="[::1]:9001" >"$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 & - sleep 40 + sleep 40 } -function start_minio_dist_erasure() -{ - export MINIO_ROOT_USER=$ACCESS_KEY - export MINIO_ROOT_PASSWORD=$SECRET_KEY - export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4" - for i in $(seq 0 3); do - "${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 & - done +function start_minio_dist_erasure() { + export MINIO_ROOT_USER=$ACCESS_KEY + export MINIO_ROOT_PASSWORD=$SECRET_KEY + export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4" + for i in $(seq 0 3); do + "${MINIO[@]}" server --address ":900${i}" >"$WORK_DIR/dist-minio-900${i}.log" 2>&1 & + done - sleep 40 + sleep 40 } -function run_test_fs() -{ - start_minio_fs +function run_test_fs() { + start_minio_fs - (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") - rv=$? + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? - pkill minio - sleep 3 + pkill minio + sleep 3 - if [ "$rv" -ne 0 ]; then - cat "$WORK_DIR/fs-minio.log" - fi - rm -f "$WORK_DIR/fs-minio.log" + if [ "$rv" -ne 0 ]; then + cat "$WORK_DIR/fs-minio.log" + fi + rm -f "$WORK_DIR/fs-minio.log" - return "$rv" + return "$rv" } -function run_test_erasure_sets() -{ - start_minio_erasure_sets +function run_test_erasure_sets() { + start_minio_erasure_sets - (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") - rv=$? + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? - pkill minio - sleep 3 + pkill minio + sleep 3 - if [ "$rv" -ne 0 ]; then - cat "$WORK_DIR/erasure-minio-sets.log" - fi - rm -f "$WORK_DIR/erasure-minio-sets.log" + if [ "$rv" -ne 0 ]; then + cat "$WORK_DIR/erasure-minio-sets.log" + fi + rm -f "$WORK_DIR/erasure-minio-sets.log" - return "$rv" + return "$rv" } -function run_test_pool_erasure_sets() -{ - start_minio_pool_erasure_sets +function run_test_pool_erasure_sets() { + start_minio_pool_erasure_sets - (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") - rv=$? + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? - pkill minio - sleep 3 + pkill minio + sleep 3 - if [ "$rv" -ne 0 ]; then - for i in $(seq 0 1); do - echo "server$i log:" - cat "$WORK_DIR/pool-minio-900$i.log" - done - fi + if [ "$rv" -ne 0 ]; then + for i in $(seq 0 1); do + echo "server$i log:" + cat "$WORK_DIR/pool-minio-900$i.log" + done + fi - for i in $(seq 0 1); do - rm -f "$WORK_DIR/pool-minio-900$i.log" - done + for i in $(seq 0 1); do + rm -f "$WORK_DIR/pool-minio-900$i.log" + done - return "$rv" + return "$rv" } -function run_test_pool_erasure_sets_ipv6() -{ - start_minio_pool_erasure_sets_ipv6 +function run_test_pool_erasure_sets_ipv6() { + start_minio_pool_erasure_sets_ipv6 - export SERVER_ENDPOINT="[::1]:9000" + export SERVER_ENDPOINT="[::1]:9000" - (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") - rv=$? + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? - pkill minio - sleep 3 + pkill minio + sleep 3 - if [ "$rv" -ne 0 ]; then - for i in $(seq 0 1); do - echo "server$i log:" - cat "$WORK_DIR/pool-minio-ipv6-900$i.log" - done - fi + if [ "$rv" -ne 0 ]; then + for i in $(seq 0 1); do + echo "server$i log:" + cat "$WORK_DIR/pool-minio-ipv6-900$i.log" + done + fi - for i in $(seq 0 1); do - rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log" - done + for i in $(seq 0 1); do + rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log" + done - return "$rv" + return "$rv" } -function run_test_erasure() -{ - start_minio_erasure +function run_test_erasure() { + start_minio_erasure - (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") - rv=$? + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? - pkill minio - sleep 3 + pkill minio + sleep 3 - if [ "$rv" -ne 0 ]; then - cat "$WORK_DIR/erasure-minio.log" - fi - rm -f "$WORK_DIR/erasure-minio.log" + if [ "$rv" -ne 0 ]; then + cat "$WORK_DIR/erasure-minio.log" + fi + rm -f "$WORK_DIR/erasure-minio.log" - return "$rv" + return "$rv" } -function run_test_dist_erasure() -{ - start_minio_dist_erasure +function run_test_dist_erasure() { + start_minio_dist_erasure - (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") - rv=$? + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? - pkill minio - sleep 3 + pkill minio + sleep 3 - if [ "$rv" -ne 0 ]; then - echo "server1 log:" - cat "$WORK_DIR/dist-minio-9000.log" - echo "server2 log:" - cat "$WORK_DIR/dist-minio-9001.log" - echo "server3 log:" - cat "$WORK_DIR/dist-minio-9002.log" - echo "server4 log:" - cat "$WORK_DIR/dist-minio-9003.log" - fi + if [ "$rv" -ne 0 ]; then + echo "server1 log:" + cat "$WORK_DIR/dist-minio-9000.log" + echo "server2 log:" + cat "$WORK_DIR/dist-minio-9001.log" + echo "server3 log:" + cat "$WORK_DIR/dist-minio-9002.log" + echo "server4 log:" + cat "$WORK_DIR/dist-minio-9003.log" + fi - rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log" + rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log" - return "$rv" + return "$rv" } -function purge() -{ - rm -rf "$1" +function purge() { + rm -rf "$1" } -function __init__() -{ - echo "Initializing environment" - mkdir -p "$WORK_DIR" - mkdir -p "$MINIO_CONFIG_DIR" - mkdir -p "$MINT_DATA_DIR" +function __init__() { + echo "Initializing environment" + mkdir -p "$WORK_DIR" + mkdir -p "$MINIO_CONFIG_DIR" + mkdir -p "$MINT_DATA_DIR" - MC_BUILD_DIR="mc-$RANDOM" - if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then - echo "failed to download https://github.com/minio/mc" - purge "${MC_BUILD_DIR}" - exit 1 - fi + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi - (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") - # remove mc source. - purge "${MC_BUILD_DIR}" + # remove mc source. + purge "${MC_BUILD_DIR}" - shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null - shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null + shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null + shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null - ## version is purposefully set to '3' for minio to migrate configuration file - echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json" + ## version is purposefully set to '3' for minio to migrate configuration file + echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json" - if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then - echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh" - exit 1 - fi + if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then + echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh" + exit 1 + fi - sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS" - chmod a+x "$FUNCTIONAL_TESTS" + sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS" + chmod a+x "$FUNCTIONAL_TESTS" } -function main() -{ - echo "Testing in FS setup" - if ! run_test_fs; then - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi +function main() { + echo "Testing in FS setup" + if ! run_test_fs; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - echo "Testing in Erasure setup" - if ! run_test_erasure; then - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + echo "Testing in Erasure setup" + if ! run_test_erasure; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - echo "Testing in Distributed Erasure setup" - if ! run_test_dist_erasure; then - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + echo "Testing in Distributed Erasure setup" + if ! run_test_dist_erasure; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - echo "Testing in Erasure setup as sets" - if ! run_test_erasure_sets; then - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + echo "Testing in Erasure setup as sets" + if ! run_test_erasure_sets; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - echo "Testing in Distributed Eraure expanded setup" - if ! run_test_pool_erasure_sets; then - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + echo "Testing in Distributed Eraure expanded setup" + if ! run_test_pool_erasure_sets; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - echo "Testing in Distributed Erasure expanded setup with ipv6" - if ! run_test_pool_erasure_sets_ipv6; then - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + echo "Testing in Distributed Erasure expanded setup with ipv6" + if ! run_test_pool_erasure_sets_ipv6; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi - purge "$WORK_DIR" + purge "$WORK_DIR" } -( __init__ "$@" && main "$@" ) +(__init__ "$@" && main "$@") rv=$? purge "$WORK_DIR" exit "$rv" diff --git a/buildscripts/verify-healing-with-root-disks.sh b/buildscripts/verify-healing-with-root-disks.sh index d8e95f325..6166eddd2 100755 --- a/buildscripts/verify-healing-with-root-disks.sh +++ b/buildscripts/verify-healing-with-root-disks.sh @@ -4,96 +4,93 @@ set -E set -o pipefail set -x - if [ ! -x "$PWD/minio" ]; then - echo "minio executable binary not found in current directory" - exit 1 + echo "minio executable binary not found in current directory" + exit 1 fi WORK_DIR="$(mktemp -d)" MINIO_CONFIG_DIR="$WORK_DIR/.minio" -MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server ) - +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) function start_minio() { - start_port=$1 + start_port=$1 - export MINIO_ROOT_USER=minio - export MINIO_ROOT_PASSWORD=minio123 - unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects - unset MINIO_CI_CD - unset CI + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + unset MINIO_CI_CD + unset CI - args=() - for i in $(seq 1 4); do - args+=("http://localhost:$[${start_port}+$i]${WORK_DIR}/mnt/disk$i/ ") - done + args=() + for i in $(seq 1 4); do + args+=("http://localhost:$((start_port + i))${WORK_DIR}/mnt/disk$i/ ") + done - for i in $(seq 1 4); do - "${MINIO[@]}" --address ":$[$start_port+$i]" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" & - done + for i in $(seq 1 4); do + "${MINIO[@]}" --address ":$((start_port + i))" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" & + done - # Wait until all nodes return 403 - for i in $(seq 1 4); do - while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$[$start_port+$i])" -ne "403" ]; do - echo -n "."; - sleep 1; - done - done + # Wait until all nodes return 403 + for i in $(seq 1 4); do + while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$((start_port + i)))" -ne "403" ]; do + echo -n "." + sleep 1 + done + done - } +} # Prepare fake disks with losetup function prepare_block_devices() { - set -e - mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/ - sudo modprobe loop - for i in 1 2 3 4; do - dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000 - device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i}) - sudo mkfs.ext4 -F ${device} - mkdir -p ${WORK_DIR}/mnt/disk${i}/ - sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/ - sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/ - done - set +e + set -e + mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/ + sudo modprobe loop + for i in 1 2 3 4; do + dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000 + device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i}) + sudo mkfs.ext4 -F ${device} + mkdir -p ${WORK_DIR}/mnt/disk${i}/ + sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/ + sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/ + done + set +e } # Start a distributed MinIO setup, unmount one disk and check if it is formatted function main() { - start_port=$(shuf -i 10000-65000 -n 1) - start_minio ${start_port} + start_port=$(shuf -i 10000-65000 -n 1) + start_minio ${start_port} - # Unmount the disk, after the unmount the device id - # /tmp/xxx/mnt/disk4 will be the same as '/' and it - # will be detected as root disk - while [ "$u" != "0" ]; do - sudo umount ${WORK_DIR}/mnt/disk4/ - u=$? - sleep 1 - done + # Unmount the disk, after the unmount the device id + # /tmp/xxx/mnt/disk4 will be the same as '/' and it + # will be detected as root disk + while [ "$u" != "0" ]; do + sudo umount ${WORK_DIR}/mnt/disk4/ + u=$? + sleep 1 + done - # Wait until MinIO self heal kicks in - sleep 60 + # Wait until MinIO self heal kicks in + sleep 60 - if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then - echo "A root disk is formatted unexpectedely" - cat "${WORK_DIR}/server4.log" - exit -1 - fi + if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then + echo "A root disk is formatted unexpectedely" + cat "${WORK_DIR}/server4.log" + exit -1 + fi } function cleanup() { - pkill minio - sudo umount ${WORK_DIR}/mnt/disk{1..3}/ - sudo rm /dev/minio-loopdisk* - rm -rf "$WORK_DIR" + pkill minio + sudo umount ${WORK_DIR}/mnt/disk{1..3}/ + sudo rm /dev/minio-loopdisk* + rm -rf "$WORK_DIR" } -( prepare_block_devices ) -( main "$@" ) +(prepare_block_devices) +(main "$@") rv=$? cleanup exit "$rv" - diff --git a/buildscripts/verify-healing.sh b/buildscripts/verify-healing.sh index 498788c22..6ede11d99 100755 --- a/buildscripts/verify-healing.sh +++ b/buildscripts/verify-healing.sh @@ -5,139 +5,135 @@ set -E set -o pipefail if [ ! -x "$PWD/minio" ]; then - echo "minio executable binary not found in current directory" - exit 1 + echo "minio executable binary not found in current directory" + exit 1 fi WORK_DIR="$PWD/.verify-$RANDOM" MINIO_CONFIG_DIR="$WORK_DIR/.minio" -MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server ) +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) function start_minio_3_node() { - export MINIO_ROOT_USER=minio - export MINIO_ROOT_PASSWORD=minio123 - export MINIO_ERASURE_SET_DRIVE_COUNT=6 - export MINIO_CI_CD=1 + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MINIO_ERASURE_SET_DRIVE_COUNT=6 + export MINIO_CI_CD=1 - start_port=$2 - args="" - for i in $(seq 1 3); do - args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/" - done - - "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 & - pid1=$! - disown ${pid1} - - "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 & - pid2=$! - disown $pid2 - - "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 & - pid3=$! - disown $pid3 - - sleep "$1" - - if ! ps -p $pid1 1>&2 > /dev/null; then - echo "server1 log:" - cat "${WORK_DIR}/dist-minio-server1.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi - - if ! ps -p $pid2 1>&2 > /dev/null; then - echo "server2 log:" - cat "${WORK_DIR}/dist-minio-server2.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi - - if ! ps -p $pid3 1>&2 > /dev/null; then - echo "server3 log:" - cat "${WORK_DIR}/dist-minio-server3.log" - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi - - if ! pkill minio; then + start_port=$2 + args="" for i in $(seq 1 3); do - echo "server$i log:" - cat "${WORK_DIR}/dist-minio-server$i.log" + args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/" done - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi - sleep 1; - if pgrep minio; then - # forcibly killing, to proceed further properly. - if ! pkill -9 minio; then - echo "no minio process running anymore, proceed." + "${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 & + pid1=$! + disown ${pid1} + + "${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 & + pid2=$! + disown $pid2 + + "${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 & + pid3=$! + disown $pid3 + + sleep "$1" + + if ! ps -p $pid1 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/dist-minio-server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 fi - fi -} + if ! ps -p $pid2 1>&2 >/dev/null; then + echo "server2 log:" + cat "${WORK_DIR}/dist-minio-server2.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + if ! ps -p $pid3 1>&2 >/dev/null; then + echo "server3 log:" + cat "${WORK_DIR}/dist-minio-server3.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + if ! pkill minio; then + for i in $(seq 1 3); do + echo "server$i log:" + cat "${WORK_DIR}/dist-minio-server$i.log" + done + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + sleep 1 + if pgrep minio; then + # forcibly killing, to proceed further properly. + if ! pkill -9 minio; then + echo "no minio process running anymore, proceed." + fi + fi +} function check_online() { - if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then - echo "1" - fi + if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then + echo "1" + fi } -function purge() -{ - rm -rf "$1" +function purge() { + rm -rf "$1" } -function __init__() -{ - echo "Initializing environment" - mkdir -p "$WORK_DIR" - mkdir -p "$MINIO_CONFIG_DIR" +function __init__() { + echo "Initializing environment" + mkdir -p "$WORK_DIR" + mkdir -p "$MINIO_CONFIG_DIR" - ## version is purposefully set to '3' for minio to migrate configuration file - echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json" + ## version is purposefully set to '3' for minio to migrate configuration file + echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json" } function perform_test() { - start_minio_3_node 120 $2 + start_minio_3_node 120 $2 - echo "Testing Distributed Erasure setup healing of drives" - echo "Remove the contents of the disks belonging to '${1}' erasure set" + echo "Testing Distributed Erasure setup healing of drives" + echo "Remove the contents of the disks belonging to '${1}' erasure set" - rm -rf ${WORK_DIR}/${1}/*/ + rm -rf ${WORK_DIR}/${1}/*/ - start_minio_3_node 120 $2 + start_minio_3_node 120 $2 - rv=$(check_online) - if [ "$rv" == "1" ]; then - for i in $(seq 1 3); do - echo "server$i log:" - cat "${WORK_DIR}/dist-minio-server$i.log" - done - pkill -9 minio - echo "FAILED" - purge "$WORK_DIR" - exit 1 - fi + rv=$(check_online) + if [ "$rv" == "1" ]; then + for i in $(seq 1 3); do + echo "server$i log:" + cat "${WORK_DIR}/dist-minio-server$i.log" + done + pkill -9 minio + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi } -function main() -{ - # use same ports for all tests - start_port=$(shuf -i 10000-65000 -n 1) +function main() { + # use same ports for all tests + start_port=$(shuf -i 10000-65000 -n 1) - perform_test "2" ${start_port} - perform_test "1" ${start_port} - perform_test "3" ${start_port} + perform_test "2" ${start_port} + perform_test "1" ${start_port} + perform_test "3" ${start_port} } -( __init__ "$@" && main "$@" ) +(__init__ "$@" && main "$@") rv=$? purge "$WORK_DIR" exit "$rv" diff --git a/docker-buildx.sh b/docker-buildx.sh index 53785c965..fb93c0f7a 100755 --- a/docker-buildx.sh +++ b/docker-buildx.sh @@ -5,21 +5,21 @@ sudo sysctl net.ipv6.conf.all.disable_ipv6=0 release=$(git describe --abbrev=0 --tags) docker buildx build --push --no-cache \ - --build-arg RELEASE="${release}" \ - -t "minio/minio:latest" \ - -t "quay.io/minio/minio:latest" \ - -t "minio/minio:${release}" \ - -t "quay.io/minio/minio:${release}" \ - --platform=linux/arm64,linux/amd64,linux/ppc64le,linux/s390x \ - -f Dockerfile.release . + --build-arg RELEASE="${release}" \ + -t "minio/minio:latest" \ + -t "quay.io/minio/minio:latest" \ + -t "minio/minio:${release}" \ + -t "quay.io/minio/minio:${release}" \ + --platform=linux/arm64,linux/amd64,linux/ppc64le,linux/s390x \ + -f Dockerfile.release . docker buildx prune -f docker buildx build --push --no-cache \ - --build-arg RELEASE="${release}" \ - -t "minio/minio:${release}.fips" \ - -t "quay.io/minio/minio:${release}.fips" \ - --platform=linux/amd64 -f Dockerfile.release.fips . + --build-arg RELEASE="${release}" \ + -t "minio/minio:${release}.fips" \ + -t "quay.io/minio/minio:${release}.fips" \ + --platform=linux/amd64 -f Dockerfile.release.fips . docker buildx prune -f diff --git a/dockerscripts/docker-entrypoint.sh b/dockerscripts/docker-entrypoint.sh index 557f8984f..0e2bdb5a2 100755 --- a/dockerscripts/docker-entrypoint.sh +++ b/dockerscripts/docker-entrypoint.sh @@ -3,26 +3,26 @@ # If command starts with an option, prepend minio. if [ "${1}" != "minio" ]; then - if [ -n "${1}" ]; then - set -- minio "$@" - fi + if [ -n "${1}" ]; then + set -- minio "$@" + fi fi # su-exec to requested user, if service cannot run exec will fail. docker_switch_user() { - if [ -n "${MINIO_USERNAME}" ] && [ -n "${MINIO_GROUPNAME}" ]; then - if [ -n "${MINIO_UID}" ] && [ -n "${MINIO_GID}" ]; then - groupadd -f -g "$MINIO_GID" "$MINIO_GROUPNAME" && \ - useradd -u "$MINIO_UID" -g "$MINIO_GROUPNAME" "$MINIO_USERNAME" - else - groupadd -f "$MINIO_GROUPNAME" && \ - useradd -g "$MINIO_GROUPNAME" "$MINIO_USERNAME" - fi - exec setpriv --reuid="${MINIO_USERNAME}" \ - --regid="${MINIO_GROUPNAME}" --keep-groups "$@" - else - exec "$@" - fi + if [ -n "${MINIO_USERNAME}" ] && [ -n "${MINIO_GROUPNAME}" ]; then + if [ -n "${MINIO_UID}" ] && [ -n "${MINIO_GID}" ]; then + groupadd -f -g "$MINIO_GID" "$MINIO_GROUPNAME" && + useradd -u "$MINIO_UID" -g "$MINIO_GROUPNAME" "$MINIO_USERNAME" + else + groupadd -f "$MINIO_GROUPNAME" && + useradd -g "$MINIO_GROUPNAME" "$MINIO_USERNAME" + fi + exec setpriv --reuid="${MINIO_USERNAME}" \ + --regid="${MINIO_GROUPNAME}" --keep-groups "$@" + else + exec "$@" + fi } ## Switch to user if applicable. diff --git a/dockerscripts/verify-minio.sh b/dockerscripts/verify-minio.sh index aa26143c3..f0f2beea7 100755 --- a/dockerscripts/verify-minio.sh +++ b/dockerscripts/verify-minio.sh @@ -4,28 +4,28 @@ set -e if [ ! -x "/opt/bin/minio" ]; then - echo "minio executable binary not found refusing to proceed" - exit 1 + echo "minio executable binary not found refusing to proceed" + exit 1 fi verify_sha256sum() { - echo "verifying binary checksum" - echo "$(awk '{print $1}' /opt/bin/minio.sha256sum) /opt/bin/minio" | sha256sum -c + echo "verifying binary checksum" + echo "$(awk '{print $1}' /opt/bin/minio.sha256sum) /opt/bin/minio" | sha256sum -c } verify_signature() { - if [ "${TARGETARCH}" = "arm" ]; then - echo "ignoring verification of binary signature" - return - fi - echo "verifying binary signature" - minisign -VQm /opt/bin/minio -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav + if [ "${TARGETARCH}" = "arm" ]; then + echo "ignoring verification of binary signature" + return + fi + echo "verifying binary signature" + minisign -VQm /opt/bin/minio -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav } main() { - verify_sha256sum + verify_sha256sum - verify_signature + verify_signature } main "$@" diff --git a/docs/bucket/replication/delete-replication.sh b/docs/bucket/replication/delete-replication.sh index 20f6527c1..68561cd38 100755 --- a/docs/bucket/replication/delete-replication.sh +++ b/docs/bucket/replication/delete-replication.sh @@ -1,26 +1,26 @@ #!/usr/bin/env bash if [ -n "$TEST_DEBUG" ]; then - set -x + set -x fi trap 'catch $LINENO' ERR # shellcheck disable=SC2120 catch() { - if [ $# -ne 0 ]; then - echo "error on line $1" - echo "dc1 server logs =========" - cat /tmp/dc1.log - echo "dc2 server logs =========" - cat /tmp/dc2.log - fi + if [ $# -ne 0 ]; then + echo "error on line $1" + echo "dc1 server logs =========" + cat /tmp/dc1.log + echo "dc2 server logs =========" + cat /tmp/dc2.log + fi - echo "Cleaning up instances of MinIO" - set +e - pkill minio - pkill mc - rm -rf /tmp/xl/ + echo "Cleaning up instances of MinIO" + set +e + pkill minio + pkill mc + rm -rf /tmp/xl/ } catch @@ -39,8 +39,8 @@ unset MINIO_KMS_KES_ENDPOINT unset MINIO_KMS_KES_KEY_NAME if [ ! -f ./mc ]; then - wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \ - chmod +x mc + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc fi mkdir -p /tmp/xl/1/ /tmp/xl/2/ @@ -49,8 +49,8 @@ export MINIO_KMS_SECRET_KEY="my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmG export MINIO_ROOT_USER="minioadmin" export MINIO_ROOT_PASSWORD="minioadmin" -./minio server --address ":9001" /tmp/xl/1/{1...4}/ 2>&1 > /tmp/dc1.log & -./minio server --address ":9002" /tmp/xl/2/{1...4}/ 2>&1 > /tmp/dc2.log & +./minio server --address ":9001" /tmp/xl/1/{1...4}/ 2>&1 >/tmp/dc1.log & +./minio server --address ":9002" /tmp/xl/2/{1...4}/ 2>&1 >/tmp/dc2.log & sleep 3 @@ -79,27 +79,27 @@ versionId="$(mc ls --json --versions myminio1/testbucket/dir/ | tail -n1 | jq -r aws s3api --endpoint-url http://localhost:9001 --profile minio delete-object --bucket testbucket --key dir/file --version-id "$versionId" -./mc ls -r --versions myminio1/testbucket > /tmp/myminio1.txt -./mc ls -r --versions myminio2/testbucket > /tmp/myminio2.txt +./mc ls -r --versions myminio1/testbucket >/tmp/myminio1.txt +./mc ls -r --versions myminio2/testbucket >/tmp/myminio2.txt out=$(diff -qpruN /tmp/myminio1.txt /tmp/myminio2.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after replication: $out" - exit 1 + echo "BUG: expected no missing entries after replication: $out" + exit 1 fi ./mc rm myminio1/testbucket/dir/file sleep 1s -./mc ls -r --versions myminio1/testbucket > /tmp/myminio1.txt -./mc ls -r --versions myminio2/testbucket > /tmp/myminio2.txt +./mc ls -r --versions myminio1/testbucket >/tmp/myminio1.txt +./mc ls -r --versions myminio2/testbucket >/tmp/myminio2.txt out=$(diff -qpruN /tmp/myminio1.txt /tmp/myminio2.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after replication: $out" - exit 1 + echo "BUG: expected no missing entries after replication: $out" + exit 1 fi echo "Success" diff --git a/docs/bucket/replication/setup_2site_existing_replication.sh b/docs/bucket/replication/setup_2site_existing_replication.sh index 211e158dd..cdcf59587 100755 --- a/docs/bucket/replication/setup_2site_existing_replication.sh +++ b/docs/bucket/replication/setup_2site_existing_replication.sh @@ -6,22 +6,22 @@ trap 'catch $LINENO' ERR # shellcheck disable=SC2120 catch() { - if [ $# -ne 0 ]; then - echo "error on line $1" - for site in sitea siteb; do - echo "$site server logs =========" - cat "/tmp/${site}_1.log" - echo "===========================" - cat "/tmp/${site}_2.log" - done - fi + if [ $# -ne 0 ]; then + echo "error on line $1" + for site in sitea siteb; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + fi - echo "Cleaning up instances of MinIO" - pkill minio - pkill -9 minio - rm -rf /tmp/multisitea - rm -rf /tmp/multisiteb - rm -rf /tmp/data + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/multisitea + rm -rf /tmp/multisiteb + rm -rf /tmp/data } catch @@ -40,19 +40,19 @@ unset MINIO_KMS_KES_ENDPOINT unset MINIO_KMS_KES_KEY_NAME if [ ! -f ./mc ]; then - wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \ - chmod +x mc + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc fi minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & sleep 10s @@ -64,7 +64,7 @@ export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9004 ## Create 100 files mkdir -p /tmp/data for i in $(seq 1 10); do - echo "T" > /tmp/data/file_${i}.txt + echo "T" >/tmp/data/file_${i}.txt done ./mc mirror /tmp/data sitea/bucket/ @@ -78,7 +78,7 @@ done echo "adding replication rule for site a -> site b" ./mc replicate add sitea/bucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket remote_arn=$(./mc replicate ls sitea/bucket --json | jq -r .rule.Destination.Bucket) sleep 1 @@ -88,20 +88,20 @@ sleep 10s ## sleep for 10s idea is that we give 100ms per object. count=$(./mc replicate resync status sitea/bucket --remote-bucket "${remote_arn}" --json | jq .resyncInfo.target[].replicationCount) -./mc ls -r --versions sitea/bucket > /tmp/sitea.txt -./mc ls -r --versions siteb/bucket > /tmp/siteb.txt +./mc ls -r --versions sitea/bucket >/tmp/sitea.txt +./mc ls -r --versions siteb/bucket >/tmp/siteb.txt out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after replication: $out" - exit 1 + echo "BUG: expected no missing entries after replication: $out" + exit 1 fi if [ $count -ne 12 ]; then - echo "resync not complete after 10s unexpected failure" - ./mc diff sitea/bucket siteb/bucket - exit 1 + echo "resync not complete after 10s unexpected failure" + ./mc diff sitea/bucket siteb/bucket + exit 1 fi ./mc cp /tmp/data/file_1.txt sitea/bucket/marker_new @@ -109,27 +109,27 @@ fi sleep 12s ## sleep for 12s idea is that we give 100ms per object. -./mc ls -r --versions sitea/bucket > /tmp/sitea.txt -./mc ls -r --versions siteb/bucket > /tmp/siteb.txt +./mc ls -r --versions sitea/bucket >/tmp/sitea.txt +./mc ls -r --versions siteb/bucket >/tmp/siteb.txt out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no 'diff' after replication: $out" - exit 1 + echo "BUG: expected no 'diff' after replication: $out" + exit 1 fi ./mc rm -r --force --versions sitea/bucket/marker sleep 14s ## sleep for 14s idea is that we give 100ms per object. -./mc ls -r --versions sitea/bucket > /tmp/sitea.txt -./mc ls -r --versions siteb/bucket > /tmp/siteb.txt +./mc ls -r --versions sitea/bucket >/tmp/sitea.txt +./mc ls -r --versions siteb/bucket >/tmp/siteb.txt out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no 'diff' after replication: $out" - exit 1 + echo "BUG: expected no 'diff' after replication: $out" + exit 1 fi ./mc mb sitea/bucket-version/ @@ -140,34 +140,34 @@ fi echo "adding replication rule for site a -> site b" ./mc replicate add sitea/bucket-version/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket-version + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket-version ./mc mb sitea/bucket-version/directory/ sleep 2s -./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt -./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt +./mc ls -r --versions sitea/bucket-version/ >/tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ >/tmp/siteb_dirs.txt out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no 'diff' after replication: $out" - exit 1 + echo "BUG: expected no 'diff' after replication: $out" + exit 1 fi ./mc rm -r --versions --force sitea/bucket-version/ sleep 2s -./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt -./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt +./mc ls -r --versions sitea/bucket-version/ >/tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ >/tmp/siteb_dirs.txt out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no 'diff' after replication: $out" - exit 1 + echo "BUG: expected no 'diff' after replication: $out" + exit 1 fi ## check if we don't create delete markers on the directory objects, its always permanent delete. @@ -179,14 +179,14 @@ sleep 2s sleep 2s -./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt -./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt +./mc ls -r --versions sitea/bucket-version/ >/tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ >/tmp/siteb_dirs.txt out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no 'diff' after replication: $out" - exit 1 + echo "BUG: expected no 'diff' after replication: $out" + exit 1 fi sitea_count=$(cat /tmp/sitea_dirs.txt | wc -l) # need to do it this way to avoid filename in the output @@ -195,13 +195,13 @@ sitea_out=$(cat /tmp/sitea_dirs.txt) siteb_out=$(cat /tmp/siteb_dirs.txt) if [ $sitea_count -ne 0 ]; then - echo "BUG: expected no 'directory objects' left after deletion: ${sitea_out}" - exit 1 + echo "BUG: expected no 'directory objects' left after deletion: ${sitea_out}" + exit 1 fi if [ $siteb_count -ne 0 ]; then - echo "BUG: expected no 'directory objects' left after deletion: ${siteb_out}" - exit 1 + echo "BUG: expected no 'directory objects' left after deletion: ${siteb_out}" + exit 1 fi catch diff --git a/docs/bucket/replication/setup_3site_replication.sh b/docs/bucket/replication/setup_3site_replication.sh index 17d38d79b..d471da3c2 100755 --- a/docs/bucket/replication/setup_3site_replication.sh +++ b/docs/bucket/replication/setup_3site_replication.sh @@ -1,29 +1,29 @@ #!/usr/bin/env bash if [ -n "$TEST_DEBUG" ]; then - set -x + set -x fi trap 'catch $LINENO' ERR # shellcheck disable=SC2120 catch() { - if [ $# -ne 0 ]; then - echo "error on line $1" - for site in sitea siteb sitec; do - echo "$site server logs =========" - cat "/tmp/${site}_1.log" - echo "===========================" - cat "/tmp/${site}_2.log" - done - fi + if [ $# -ne 0 ]; then + echo "error on line $1" + for site in sitea siteb sitec; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + fi - echo "Cleaning up instances of MinIO" - pkill minio - pkill -9 minio - rm -rf /tmp/multisitea - rm -rf /tmp/multisiteb - rm -rf /tmp/multisitec + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/multisitea + rm -rf /tmp/multisiteb + rm -rf /tmp/multisitec } catch @@ -42,25 +42,25 @@ unset MINIO_KMS_KES_ENDPOINT unset MINIO_KMS_KES_KEY_NAME go build ./docs/debugging/s3-check-md5/ -wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \ - && chmod +x mc -wget -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z \ - && chmod +x mc.RELEASE.2021-03-12T03-36-59Z +wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +wget -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z && + chmod +x mc.RELEASE.2021-03-12T03-36-59Z minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & minio server --address 127.0.0.1:9005 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 & + "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 & minio server --address 127.0.0.1:9006 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 & + "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 & sleep 30 @@ -83,74 +83,74 @@ export MC_HOST_sitec=http://minio:minio123@127.0.0.1:9006 echo "adding replication rule for a -> b : ${remote_arn}" sleep 1 ./mc replicate add sitea/bucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" sleep 1 echo "adding replication rule for b -> a : ${remote_arn}" ./mc replicate add siteb/bucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" + --remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" sleep 1 echo "adding replication rule for a -> c : ${remote_arn}" ./mc replicate add sitea/bucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9006/bucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 + --remote-bucket http://minio:minio123@127.0.0.1:9006/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 sleep 1 echo "adding replication rule for c -> a : ${remote_arn}" ./mc replicate add sitec/bucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 + --remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 sleep 1 echo "adding replication rule for b -> c : ${remote_arn}" ./mc replicate add siteb/bucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9006/bucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 + --remote-bucket http://minio:minio123@127.0.0.1:9006/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 sleep 1 echo "adding replication rule for c -> b : ${remote_arn}" ./mc replicate add sitec/bucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 sleep 1 echo "adding replication rule for olockbucket a -> b : ${remote_arn}" ./mc replicate add sitea/olockbucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9004/olockbucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" + --remote-bucket http://minio:minio123@127.0.0.1:9004/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" sleep 1 echo "adding replication rule for olockbucket b -> a : ${remote_arn}" ./mc replicate add siteb/olockbucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9001/olockbucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" + --remote-bucket http://minio:minio123@127.0.0.1:9001/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" sleep 1 echo "adding replication rule for olockbucket a -> c : ${remote_arn}" ./mc replicate add sitea/olockbucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9006/olockbucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 + --remote-bucket http://minio:minio123@127.0.0.1:9006/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 sleep 1 echo "adding replication rule for olockbucket c -> a : ${remote_arn}" ./mc replicate add sitec/olockbucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9001/olockbucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 + --remote-bucket http://minio:minio123@127.0.0.1:9001/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 sleep 1 echo "adding replication rule for olockbucket b -> c : ${remote_arn}" ./mc replicate add siteb/olockbucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9006/olockbucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 + --remote-bucket http://minio:minio123@127.0.0.1:9006/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 sleep 1 echo "adding replication rule for olockbucket c -> b : ${remote_arn}" ./mc replicate add sitec/olockbucket/ \ - --remote-bucket http://minio:minio123@127.0.0.1:9004/olockbucket \ - --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 + --remote-bucket http://minio:minio123@127.0.0.1:9004/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 sleep 1 echo "Set default governance retention 30d" @@ -166,20 +166,20 @@ sleep 1 echo "Verifying the metadata difference between source and target" if diff -pruN <(./mc stat --json sitea/bucket/hosts | jq .) <(./mc stat --json siteb/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then - echo "verified sitea-> COMPLETED, siteb-> REPLICA" + echo "verified sitea-> COMPLETED, siteb-> REPLICA" fi if diff -pruN <(./mc stat --json sitea/bucket/hosts | jq .) <(./mc stat --json sitec/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then - echo "verified sitea-> COMPLETED, sitec-> REPLICA" + echo "verified sitea-> COMPLETED, sitec-> REPLICA" fi echo "Verifying the metadata difference between source and target" if diff -pruN <(./mc stat --json sitea/olockbucket/hosts | jq .) <(./mc stat --json siteb/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then - echo "verified sitea-> COMPLETED, siteb-> REPLICA" + echo "verified sitea-> COMPLETED, siteb-> REPLICA" fi if diff -pruN <(./mc stat --json sitea/olockbucket/hosts | jq .) <(./mc stat --json sitec/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then - echo "verified sitea-> COMPLETED, sitec-> REPLICA" + echo "verified sitea-> COMPLETED, sitec-> REPLICA" fi sleep 5 diff --git a/docs/bucket/replication/setup_replication.sh b/docs/bucket/replication/setup_replication.sh index d9b5803eb..1b3774a83 100755 --- a/docs/bucket/replication/setup_replication.sh +++ b/docs/bucket/replication/setup_replication.sh @@ -9,7 +9,7 @@ mc mb -l dest/bucket mc admin user add source repladmin repladmin123 # create a replication policy for repladmin -cat > repladmin-policy-source.json <repladmin-policy-source.json < replpolicy.json <replpolicy.json <&1 >/dev/null)& +(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & pid=$! sleep 2 @@ -61,38 +61,37 @@ expanded_user_count=$(./mc admin user list myminio/ | wc -l) expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) if [ $user_count -ne $expanded_user_count ]; then - echo "BUG: original user count differs from expanded setup" - exit 1 + echo "BUG: original user count differs from expanded setup" + exit 1 fi if [ $policy_count -ne $expanded_policy_count ]; then - echo "BUG: original policy count differs from expanded setup" - exit 1 + echo "BUG: original policy count differs from expanded setup" + exit 1 fi ./mc version info myminio/versioned | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "expected versioning enabled after expansion" - exit 1 + echo "expected versioning enabled after expansion" + exit 1 fi ./mc mirror cmd myminio/versioned/ --quiet >/dev/null -./mc ls -r myminio/versioned/ > expanded_ns.txt -./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt ./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1} -until $(./mc admin decom status myminio/ | grep -q Complete) -do - echo "waiting for decom to finish..." - sleep 1 +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + sleep 1 done kill $pid -(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log)& +(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) & pid=$! sleep 2 @@ -101,43 +100,43 @@ decom_user_count=$(./mc admin user list myminio/ | wc -l) decom_policy_count=$(./mc admin policy list myminio/ | wc -l) if [ $user_count -ne $decom_user_count ]; then - echo "BUG: original user count differs after decommission" - exit 1 + echo "BUG: original user count differs after decommission" + exit 1 fi if [ $policy_count -ne $decom_policy_count ]; then - echo "BUG: original policy count differs after decommission" - exit 1 + echo "BUG: original policy count differs after decommission" + exit 1 fi ./mc version info myminio/versioned | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected versioning enabled after decommission" - exit 1 + echo "BUG: expected versioning enabled after decommission" + exit 1 fi got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) if [ "${expected_checksum}" != "${got_checksum}" ]; then - echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" - exit 1 + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 fi -./mc ls -r myminio/versioned > decommissioned_ns.txt -./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission: $out" - exit 1 + echo "BUG: expected no missing entries after decommission: $out" + exit 1 fi out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission: $out" - exit 1 + echo "BUG: expected no missing entries after decommission: $out" + exit 1 fi # kill $pid diff --git a/docs/distributed/decom-encrypted-sse-s3.sh b/docs/distributed/decom-encrypted-sse-s3.sh index 0f58573c0..76223cadf 100644 --- a/docs/distributed/decom-encrypted-sse-s3.sh +++ b/docs/distributed/decom-encrypted-sse-s3.sh @@ -1,22 +1,22 @@ #!/bin/bash if [ -n "$TEST_DEBUG" ]; then - set -x + set -x fi pkill minio rm -rf /tmp/xl if [ ! -f ./mc ]; then - wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \ - chmod +x mc + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc fi export CI=true export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/" -(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)& +(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & pid=$! sleep 2 @@ -58,45 +58,44 @@ expanded_user_count=$(./mc admin user list myminio/ | wc -l) expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) if [ $user_count -ne $expanded_user_count ]; then - echo "BUG: original user count differs from expanded setup" - exit 1 + echo "BUG: original user count differs from expanded setup" + exit 1 fi if [ $policy_count -ne $expanded_policy_count ]; then - echo "BUG: original policy count differs from expanded setup" - exit 1 + echo "BUG: original policy count differs from expanded setup" + exit 1 fi ./mc version info myminio/versioned | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "expected versioning enabled after expansion" - exit 1 + echo "expected versioning enabled after expansion" + exit 1 fi ./mc encrypt info myminio/versioned | grep -q "Auto encryption 'sse-s3' is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "expected encryption enabled after expansion" - exit 1 + echo "expected encryption enabled after expansion" + exit 1 fi ./mc mirror cmd myminio/versioned/ --quiet >/dev/null -./mc ls -r myminio/versioned/ > expanded_ns.txt -./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt ./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1} -until $(./mc admin decom status myminio/ | grep -q Complete) -do - echo "waiting for decom to finish..." - sleep 1 +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + sleep 1 done kill $pid -(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log)& +(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) & pid=$! sleep 2 @@ -105,50 +104,50 @@ decom_user_count=$(./mc admin user list myminio/ | wc -l) decom_policy_count=$(./mc admin policy list myminio/ | wc -l) if [ $user_count -ne $decom_user_count ]; then - echo "BUG: original user count differs after decommission" - exit 1 + echo "BUG: original user count differs after decommission" + exit 1 fi if [ $policy_count -ne $decom_policy_count ]; then - echo "BUG: original policy count differs after decommission" - exit 1 + echo "BUG: original policy count differs after decommission" + exit 1 fi ./mc version info myminio/versioned | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected versioning enabled after decommission" - exit 1 + echo "BUG: expected versioning enabled after decommission" + exit 1 fi ./mc encrypt info myminio/versioned | grep -q "Auto encryption 'sse-s3' is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected encryption enabled after expansion" - exit 1 + echo "BUG: expected encryption enabled after expansion" + exit 1 fi got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) if [ "${expected_checksum}" != "${got_checksum}" ]; then - echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" - exit 1 + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 fi -./mc ls -r myminio/versioned > decommissioned_ns.txt -./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission: $out" - exit 1 + echo "BUG: expected no missing entries after decommission: $out" + exit 1 fi out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission: $out" - exit 1 + echo "BUG: expected no missing entries after decommission: $out" + exit 1 fi kill $pid diff --git a/docs/distributed/decom-encrypted.sh b/docs/distributed/decom-encrypted.sh index 779d59225..5e1a2167f 100644 --- a/docs/distributed/decom-encrypted.sh +++ b/docs/distributed/decom-encrypted.sh @@ -1,22 +1,22 @@ #!/bin/bash if [ -n "$TEST_DEBUG" ]; then - set -x + set -x fi pkill minio rm -rf /tmp/xl if [ ! -f ./mc ]; then - wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \ - chmod +x mc + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc fi export CI=true export MINIO_KMS_AUTO_ENCRYPTION=on export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= -(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)& +(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & pid=$! sleep 2 @@ -57,38 +57,37 @@ expanded_user_count=$(./mc admin user list myminio/ | wc -l) expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) if [ $user_count -ne $expanded_user_count ]; then - echo "BUG: original user count differs from expanded setup" - exit 1 + echo "BUG: original user count differs from expanded setup" + exit 1 fi if [ $policy_count -ne $expanded_policy_count ]; then - echo "BUG: original policy count differs from expanded setup" - exit 1 + echo "BUG: original policy count differs from expanded setup" + exit 1 fi ./mc version info myminio/versioned | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "expected versioning enabled after expansion" - exit 1 + echo "expected versioning enabled after expansion" + exit 1 fi ./mc mirror cmd myminio/versioned/ --quiet >/dev/null -./mc ls -r myminio/versioned/ > expanded_ns.txt -./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt ./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1} -until $(./mc admin decom status myminio/ | grep -q Complete) -do - echo "waiting for decom to finish..." - sleep 1 +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + sleep 1 done kill $pid -(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/dev/null)& +(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/dev/null) & pid=$! sleep 2 @@ -97,43 +96,43 @@ decom_user_count=$(./mc admin user list myminio/ | wc -l) decom_policy_count=$(./mc admin policy list myminio/ | wc -l) if [ $user_count -ne $decom_user_count ]; then - echo "BUG: original user count differs after decommission" - exit 1 + echo "BUG: original user count differs after decommission" + exit 1 fi if [ $policy_count -ne $decom_policy_count ]; then - echo "BUG: original policy count differs after decommission" - exit 1 + echo "BUG: original policy count differs after decommission" + exit 1 fi ./mc version info myminio/versioned | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected versioning enabled after decommission" - exit 1 + echo "BUG: expected versioning enabled after decommission" + exit 1 fi -./mc ls -r myminio/versioned > decommissioned_ns.txt -./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission: $out" - exit 1 + echo "BUG: expected no missing entries after decommission: $out" + exit 1 fi out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission: $out" - exit 1 + echo "BUG: expected no missing entries after decommission: $out" + exit 1 fi got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) if [ "${expected_checksum}" != "${got_checksum}" ]; then - echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" - exit 1 + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 fi kill $pid diff --git a/docs/distributed/decom.sh b/docs/distributed/decom.sh index fed4c0dc0..21a6cd64a 100755 --- a/docs/distributed/decom.sh +++ b/docs/distributed/decom.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -n "$TEST_DEBUG" ]; then - set -x + set -x fi pkill minio @@ -9,13 +9,13 @@ rm -rf /tmp/xl rm -rf /tmp/xltier if [ ! -f ./mc ]; then - wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \ - chmod +x mc + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc fi export CI=true -(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)& +(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & pid=$! sleep 2 @@ -47,7 +47,7 @@ user_count=$(./mc admin user list myminio/ | wc -l) policy_count=$(./mc admin policy list myminio/ | wc -l) ## create a warm tier instance -(minio server /tmp/xltier/{1...4}/disk{0...1} --address :9001 2>&1 >/dev/null)& +(minio server /tmp/xltier/{1...4}/disk{0...1} --address :9001 2>&1 >/dev/null) & sleep 2 export MC_HOST_mytier="http://minioadmin:minioadmin@localhost:9001/" @@ -58,10 +58,10 @@ export MC_HOST_mytier="http://minioadmin:minioadmin@localhost:9001/" ./mc ilm add myminio/bucket2 --transition-days 0 --transition-tier TIER1 --transition-days 0 ## mirror some content to bucket2 and capture versions tiered ./mc mirror internal myminio/bucket2/ --quiet >/dev/null -./mc ls -r myminio/bucket2/ > bucket2_ns.txt -./mc ls -r --versions myminio/bucket2/ > bucket2_ns_versions.txt +./mc ls -r myminio/bucket2/ >bucket2_ns.txt +./mc ls -r --versions myminio/bucket2/ >bucket2_ns_versions.txt sleep 2 -./mc ls -r --versions mytier/tiered/ > tiered_ns_versions.txt +./mc ls -r --versions mytier/tiered/ >tiered_ns_versions.txt kill $pid (minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) & @@ -73,38 +73,37 @@ expanded_user_count=$(./mc admin user list myminio/ | wc -l) expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) if [ $user_count -ne $expanded_user_count ]; then - echo "BUG: original user count differs from expanded setup" - exit 1 + echo "BUG: original user count differs from expanded setup" + exit 1 fi if [ $policy_count -ne $expanded_policy_count ]; then - echo "BUG: original policy count differs from expanded setup" - exit 1 + echo "BUG: original policy count differs from expanded setup" + exit 1 fi ./mc version info myminio/versioned | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "expected versioning enabled after expansion" - exit 1 + echo "expected versioning enabled after expansion" + exit 1 fi ./mc mirror cmd myminio/versioned/ --quiet >/dev/null -./mc ls -r myminio/versioned/ > expanded_ns.txt -./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt ./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1} -until $(./mc admin decom status myminio/ | grep -q Complete) -do - echo "waiting for decom to finish..." - sleep 1 +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + sleep 1 done kill $pid -(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/dev/null)& +(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/dev/null) & pid=$! sleep 2 @@ -113,82 +112,82 @@ decom_user_count=$(./mc admin user list myminio/ | wc -l) decom_policy_count=$(./mc admin policy list myminio/ | wc -l) if [ $user_count -ne $decom_user_count ]; then - echo "BUG: original user count differs after decommission" - exit 1 + echo "BUG: original user count differs after decommission" + exit 1 fi if [ $policy_count -ne $decom_policy_count ]; then - echo "BUG: original policy count differs after decommission" - exit 1 + echo "BUG: original policy count differs after decommission" + exit 1 fi ./mc version info myminio/versioned | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected versioning enabled after decommission" - exit 1 + echo "BUG: expected versioning enabled after decommission" + exit 1 fi -./mc ls -r myminio/versioned > decommissioned_ns.txt -./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission: $out" - exit 1 + echo "BUG: expected no missing entries after decommission: $out" + exit 1 fi out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission: $out" - exit 1 + echo "BUG: expected no missing entries after decommission: $out" + exit 1 fi got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) if [ "${expected_checksum}" != "${got_checksum}" ]; then - echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" - exit 1 + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 fi # after decommissioning, compare listings in bucket2 and tiered ./mc version info myminio/bucket2 | grep -q "versioning is enabled" ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected versioning enabled after decommission on bucket2" - exit 1 + echo "BUG: expected versioning enabled after decommission on bucket2" + exit 1 fi -./mc ls -r myminio/bucket2 > decommissioned_bucket2_ns.txt -./mc ls -r --versions myminio/bucket2 > decommissioned_bucket2_ns_versions.txt -./mc ls -r --versions mytier/tiered/ > tiered_ns_versions2.txt +./mc ls -r myminio/bucket2 >decommissioned_bucket2_ns.txt +./mc ls -r --versions myminio/bucket2 >decommissioned_bucket2_ns_versions.txt +./mc ls -r --versions mytier/tiered/ >tiered_ns_versions2.txt out=$(diff -qpruN bucket2_ns.txt decommissioned_bucket2_ns.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission in bucket2: $out" - exit 1 + echo "BUG: expected no missing entries after decommission in bucket2: $out" + exit 1 fi out=$(diff -qpruN bucket2_ns_versions.txt decommissioned_bucket2_ns_versions.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission in bucket2x: $out" - exit 1 + echo "BUG: expected no missing entries after decommission in bucket2x: $out" + exit 1 fi out=$(diff -qpruN tiered_ns_versions.txt tiered_ns_versions2.txt) ret=$? if [ $ret -ne 0 ]; then - echo "BUG: expected no missing entries after decommission in warm tier: $out" - exit 1 + echo "BUG: expected no missing entries after decommission in warm tier: $out" + exit 1 fi got_checksum=$(./mc cat myminio/bucket2/dsync/drwmutex.go | md5sum) if [ "${expected_checksum}" != "${got_checksum}" ]; then - echo "BUG: decommission failed on encrypted objects with tiering: expected ${expected_checksum} got ${got_checksum}" - exit 1 + echo "BUG: decommission failed on encrypted objects with tiering: expected ${expected_checksum} got ${got_checksum}" + exit 1 fi kill $pid diff --git a/docs/site-replication/run-multi-site-ldap.sh b/docs/site-replication/run-multi-site-ldap.sh index 2c881a5a7..37b5e6cca 100755 --- a/docs/site-replication/run-multi-site-ldap.sh +++ b/docs/site-replication/run-multi-site-ldap.sh @@ -2,23 +2,23 @@ # shellcheck disable=SC2120 exit_1() { - cleanup + cleanup - echo "minio1 ============" - cat /tmp/minio1_1.log - echo "minio2 ============" - cat /tmp/minio2_1.log - echo "minio3 ============" - cat /tmp/minio3_1.log + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + echo "minio3 ============" + cat /tmp/minio3_1.log - exit 1 + exit 1 } cleanup() { - echo "Cleaning up instances of MinIO" - pkill minio - pkill -9 minio - rm -rf /tmp/minio-ldap-idp{1,2,3} + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/minio-ldap-idp{1,2,3} } cleanup @@ -45,8 +45,8 @@ export MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN="ou=swengg,dc=min,dc=io" export MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER="(&(objectclass=groupOfNames)(member=%d))" if [ ! -f ./mc ]; then - wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \ - && chmod +x mc + wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc fi minio server --config-dir /tmp/minio-ldap --address ":9001" /tmp/minio-ldap-idp1/{1...4} >/tmp/minio1_1.log 2>&1 & @@ -80,72 +80,72 @@ sleep 5 sleep 10 ./mc admin policy info minio1 rw if [ $? -eq 0 ]; then - echo "expecting the command to fail, exiting.." - exit_1; + echo "expecting the command to fail, exiting.." + exit_1 fi ./mc admin policy info minio2 rw if [ $? -eq 0 ]; then - echo "expecting the command to fail, exiting.." - exit_1; + echo "expecting the command to fail, exiting.." + exit_1 fi ./mc admin user info minio1 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" if [ $? -ne 0 ]; then - echo "policy mapping missing, exiting.." - exit_1; + echo "policy mapping missing, exiting.." + exit_1 fi ./mc admin user info minio2 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" if [ $? -ne 0 ]; then - echo "policy mapping missing, exiting.." - exit_1; + echo "policy mapping missing, exiting.." + exit_1 fi ./mc admin user info minio3 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" if [ $? -ne 0 ]; then - echo "policy mapping missing, exiting.." - exit_1; + echo "policy mapping missing, exiting.." + exit_1 fi # LDAP simple user ./mc admin user svcacct add minio2 dillon --access-key testsvc --secret-key testsvc123 if [ $? -ne 0 ]; then - echo "adding svc account failed, exiting.." - exit_1; + echo "adding svc account failed, exiting.." + exit_1 fi sleep 10 ./mc admin user svcacct info minio1 testsvc if [ $? -ne 0 ]; then - echo "svc account not mirrored, exiting.." - exit_1; + echo "svc account not mirrored, exiting.." + exit_1 fi ./mc admin user svcacct info minio2 testsvc if [ $? -ne 0 ]; then - echo "svc account not mirrored, exiting.." - exit_1; + echo "svc account not mirrored, exiting.." + exit_1 fi ./mc admin user svcacct rm minio1 testsvc if [ $? -ne 0 ]; then - echo "removing svc account failed, exiting.." - exit_1; + echo "removing svc account failed, exiting.." + exit_1 fi sleep 10 ./mc admin user svcacct info minio2 testsvc if [ $? -eq 0 ]; then - echo "svc account found after delete, exiting.." - exit_1; + echo "svc account found after delete, exiting.." + exit_1 fi ./mc admin user svcacct info minio3 testsvc if [ $? -eq 0 ]; then - echo "svc account found after delete, exiting.." - exit_1; + echo "svc account found after delete, exiting.." + exit_1 fi ./mc mb minio1/newbucket @@ -161,14 +161,14 @@ expected_checksum=$(cat ./lrgfile | md5sum) sleep 5 ./mc stat minio2/newbucket if [ $? -ne 0 ]; then - echo "expecting bucket to be present. exiting.." - exit_1; + echo "expecting bucket to be present. exiting.." + exit_1 fi ./mc stat minio3/newbucket if [ $? -ne 0 ]; then - echo "expecting bucket to be present. exiting.." - exit_1; + echo "expecting bucket to be present. exiting.." + exit_1 fi ./mc cp README.md minio2/newbucket/ @@ -176,57 +176,57 @@ fi sleep 5 ./mc stat minio1/newbucket/README.md if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi ./mc stat minio3/newbucket/README.md if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi sleep 10 ./mc stat minio3/newbucket/lrgfile if [ $? -ne 0 ]; then - echo "expected object to be present, exiting.." - exit_1; + echo "expected object to be present, exiting.." + exit_1 fi actual_checksum=$(./mc cat minio3/newbucket/lrgfile | md5sum) if [ "${expected_checksum}" != "${actual_checksum}" ]; then - echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" - exit + echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" + exit fi rm ./lrgfile vID=$(./mc stat minio2/newbucket/README.md --json | jq .versionID) if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi ./mc tag set --version-id "${vID}" minio2/newbucket/README.md "k=v" if [ $? -ne 0 ]; then - echo "expecting tag set to be successful. exiting.." - exit_1; + echo "expecting tag set to be successful. exiting.." + exit_1 fi sleep 5 ./mc tag remove --version-id "${vID}" minio2/newbucket/README.md if [ $? -ne 0 ]; then - echo "expecting tag removal to be successful. exiting.." - exit_1; + echo "expecting tag removal to be successful. exiting.." + exit_1 fi sleep 5 replStatus_minio2=$(./mc stat minio2/newbucket/README.md --json | jq -r .replicationStatus) if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi if [ ${replStatus_minio2} != "COMPLETED" ]; then - echo "expected tag removal to have replicated, exiting..." - exit_1; + echo "expected tag removal to have replicated, exiting..." + exit_1 fi ./mc rm minio3/newbucket/README.md @@ -234,54 +234,54 @@ sleep 5 ./mc stat minio2/newbucket/README.md if [ $? -eq 0 ]; then - echo "expected file to be deleted, exiting.." - exit_1; + echo "expected file to be deleted, exiting.." + exit_1 fi ./mc stat minio1/newbucket/README.md if [ $? -eq 0 ]; then - echo "expected file to be deleted, exiting.." - exit_1; + echo "expected file to be deleted, exiting.." + exit_1 fi ./mc mb --with-lock minio3/newbucket-olock sleep 5 -enabled_minio2=$(./mc stat --json minio2/newbucket-olock| jq -r .ObjectLock.enabled) +enabled_minio2=$(./mc stat --json minio2/newbucket-olock | jq -r .ObjectLock.enabled) if [ $? -ne 0 ]; then - echo "expected bucket to be mirrored with object-lock but not present, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 fi if [ "${enabled_minio2}" != "Enabled" ]; then - echo "expected bucket to be mirrored with object-lock enabled, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 fi -enabled_minio1=$(./mc stat --json minio1/newbucket-olock| jq -r .ObjectLock.enabled) +enabled_minio1=$(./mc stat --json minio1/newbucket-olock | jq -r .ObjectLock.enabled) if [ $? -ne 0 ]; then - echo "expected bucket to be mirrored with object-lock but not present, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 fi if [ "${enabled_minio1}" != "Enabled" ]; then - echo "expected bucket to be mirrored with object-lock enabled, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 fi # "Test if most recent tag update is replicated" ./mc tag set minio2/newbucket "key=val1" if [ $? -ne 0 ]; then - echo "expecting tag set to be successful. exiting.." - exit_1; + echo "expecting tag set to be successful. exiting.." + exit_1 fi sleep 10 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) if [ "${val}" != "val1" ]; then - echo "expected bucket tag to have replicated, exiting..." - exit_1; + echo "expected bucket tag to have replicated, exiting..." + exit_1 fi # stop minio1 kill -9 ${site1_pid} @@ -297,17 +297,17 @@ minio server --config-dir /tmp/minio-ldap --address ":9001" /tmp/minio-ldap-idp1 sleep 200 # Test whether most recent tag update on minio2 is replicated to minio1 -val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key ) +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) if [ "${val}" != "val2" ]; then - echo "expected bucket tag to have replicated, exiting..." - exit_1; + echo "expected bucket tag to have replicated, exiting..." + exit_1 fi # Test if bucket created/deleted when minio1 is down healed -diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null -if [ $? -ne 0 ]; then - echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." - exit_1; +diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null +if [ $? -ne 0 ]; then + echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." + exit_1 fi cleanup diff --git a/docs/site-replication/run-multi-site-minio-idp.sh b/docs/site-replication/run-multi-site-minio-idp.sh index 71625c38b..02d3000f4 100755 --- a/docs/site-replication/run-multi-site-minio-idp.sh +++ b/docs/site-replication/run-multi-site-minio-idp.sh @@ -2,23 +2,23 @@ # shellcheck disable=SC2120 exit_1() { - cleanup + cleanup - echo "minio1 ============" - cat /tmp/minio1_1.log - echo "minio2 ============" - cat /tmp/minio2_1.log - echo "minio3 ============" - cat /tmp/minio3_1.log + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + echo "minio3 ============" + cat /tmp/minio3_1.log - exit 1 + exit 1 } cleanup() { - echo "Cleaning up instances of MinIO" - pkill minio - pkill -9 minio - rm -rf /tmp/minio-internal-idp{1,2,3} + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/minio-internal-idp{1,2,3} } cleanup @@ -37,8 +37,8 @@ export MINIO_PROMETHEUS_AUTH_TYPE=public export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= if [ ! -f ./mc ]; then - wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \ - && chmod +x mc + wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc fi minio server --config-dir /tmp/minio-internal --address ":9001" /tmp/minio-internal-idp1/{1...4} >/tmp/minio1_1.log 2>&1 & @@ -90,83 +90,83 @@ sleep 10 ./mc admin policy info minio1 rw if [ $? -eq 0 ]; then - echo "expecting the command to fail, exiting.." - exit_1; + echo "expecting the command to fail, exiting.." + exit_1 fi ./mc admin policy info minio2 rw if [ $? -eq 0 ]; then - echo "expecting the command to fail, exiting.." - exit_1; + echo "expecting the command to fail, exiting.." + exit_1 fi ./mc admin policy info minio3 rw if [ $? -eq 0 ]; then - echo "expecting the command to fail, exiting.." - exit_1; + echo "expecting the command to fail, exiting.." + exit_1 fi ./mc admin user info minio1 foobar if [ $? -ne 0 ]; then - echo "policy mapping missing on 'minio1', exiting.." - exit_1; + echo "policy mapping missing on 'minio1', exiting.." + exit_1 fi ./mc admin user info minio2 foobar if [ $? -ne 0 ]; then - echo "policy mapping missing on 'minio2', exiting.." - exit_1; + echo "policy mapping missing on 'minio2', exiting.." + exit_1 fi ./mc admin user info minio3 foobar if [ $? -ne 0 ]; then - echo "policy mapping missing on 'minio3', exiting.." - exit_1; + echo "policy mapping missing on 'minio3', exiting.." + exit_1 fi ./mc admin group info minio3 foobar-g if [ $? -ne 0 ]; then - echo "group mapping missing on 'minio3', exiting.." - exit_1; + echo "group mapping missing on 'minio3', exiting.." + exit_1 fi ./mc admin user svcacct add minio2 foobar --access-key testsvc --secret-key testsvc123 if [ $? -ne 0 ]; then - echo "adding svc account failed, exiting.." - exit_1; + echo "adding svc account failed, exiting.." + exit_1 fi sleep 10 ./mc admin user svcacct info minio1 testsvc if [ $? -ne 0 ]; then - echo "svc account not mirrored, exiting.." - exit_1; + echo "svc account not mirrored, exiting.." + exit_1 fi ./mc admin user svcacct info minio2 testsvc if [ $? -ne 0 ]; then - echo "svc account not mirrored, exiting.." - exit_1; + echo "svc account not mirrored, exiting.." + exit_1 fi ./mc admin user svcacct rm minio1 testsvc if [ $? -ne 0 ]; then - echo "removing svc account failed, exiting.." - exit_1; + echo "removing svc account failed, exiting.." + exit_1 fi sleep 10 ./mc admin user svcacct info minio2 testsvc if [ $? -eq 0 ]; then - echo "svc account found after delete, exiting.." - exit_1; + echo "svc account found after delete, exiting.." + exit_1 fi ./mc admin user svcacct info minio3 testsvc if [ $? -eq 0 ]; then - echo "svc account found after delete, exiting.." - exit_1; + echo "svc account found after delete, exiting.." + exit_1 fi ./mc mb minio1/newbucket @@ -179,25 +179,25 @@ expected_checksum=$(cat ./lrgfile | md5sum) sleep 5 ./mc stat minio2/newbucket if [ $? -ne 0 ]; then - echo "expecting bucket to be present. exiting.." - exit_1; + echo "expecting bucket to be present. exiting.." + exit_1 fi ./mc stat minio3/newbucket if [ $? -ne 0 ]; then - echo "expecting bucket to be present. exiting.." - exit_1; + echo "expecting bucket to be present. exiting.." + exit_1 fi err_minio2=$(./mc stat minio2/newbucket/xxx --json | jq -r .error.cause.message) if [ $? -ne 0 ]; then - echo "expecting object to be missing. exiting.." - exit_1; + echo "expecting object to be missing. exiting.." + exit_1 fi if [ "${err_minio2}" != "Object does not exist" ]; then - echo "expected to see Object does not exist error, exiting..." - exit_1; + echo "expected to see Object does not exist error, exiting..." + exit_1 fi ./mc cp README.md minio2/newbucket/ @@ -205,58 +205,58 @@ fi sleep 5 ./mc stat minio1/newbucket/README.md if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi ./mc stat minio3/newbucket/README.md if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi sleep 10 ./mc stat minio3/newbucket/lrgfile if [ $? -ne 0 ]; then - echo "expected object to be present, exiting.." - exit_1; + echo "expected object to be present, exiting.." + exit_1 fi actual_checksum=$(./mc cat minio3/newbucket/lrgfile | md5sum) if [ "${expected_checksum}" != "${actual_checksum}" ]; then - echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" - exit + echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" + exit fi rm ./lrgfile vID=$(./mc stat minio2/newbucket/README.md --json | jq .versionID) if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi ./mc tag set --version-id "${vID}" minio2/newbucket/README.md "k=v" if [ $? -ne 0 ]; then - echo "expecting tag set to be successful. exiting.." - exit_1; + echo "expecting tag set to be successful. exiting.." + exit_1 fi sleep 5 ./mc tag remove --version-id "${vID}" minio2/newbucket/README.md if [ $? -ne 0 ]; then - echo "expecting tag removal to be successful. exiting.." - exit_1; + echo "expecting tag removal to be successful. exiting.." + exit_1 fi sleep 5 -replStatus_minio2=$(./mc stat minio2/newbucket/README.md --json | jq -r .replicationStatus ) +replStatus_minio2=$(./mc stat minio2/newbucket/README.md --json | jq -r .replicationStatus) if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi if [ ${replStatus_minio2} != "COMPLETED" ]; then - echo "expected tag removal to have replicated, exiting..." - exit_1; + echo "expected tag removal to have replicated, exiting..." + exit_1 fi ./mc rm minio3/newbucket/README.md @@ -264,72 +264,72 @@ sleep 5 ./mc stat minio2/newbucket/README.md if [ $? -eq 0 ]; then - echo "expected file to be deleted, exiting.." - exit_1; + echo "expected file to be deleted, exiting.." + exit_1 fi ./mc stat minio1/newbucket/README.md if [ $? -eq 0 ]; then - echo "expected file to be deleted, exiting.." - exit_1; + echo "expected file to be deleted, exiting.." + exit_1 fi ./mc mb --with-lock minio3/newbucket-olock sleep 5 -enabled_minio2=$(./mc stat --json minio2/newbucket-olock| jq -r .ObjectLock.enabled) +enabled_minio2=$(./mc stat --json minio2/newbucket-olock | jq -r .ObjectLock.enabled) if [ $? -ne 0 ]; then - echo "expected bucket to be mirrored with object-lock but not present, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 fi if [ "${enabled_minio2}" != "Enabled" ]; then - echo "expected bucket to be mirrored with object-lock enabled, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 fi -enabled_minio1=$(./mc stat --json minio1/newbucket-olock| jq -r .ObjectLock.enabled) +enabled_minio1=$(./mc stat --json minio1/newbucket-olock | jq -r .ObjectLock.enabled) if [ $? -ne 0 ]; then - echo "expected bucket to be mirrored with object-lock but not present, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 fi if [ "${enabled_minio1}" != "Enabled" ]; then - echo "expected bucket to be mirrored with object-lock enabled, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 fi # "Test if most recent tag update is replicated" ./mc tag set minio2/newbucket "key=val1" if [ $? -ne 0 ]; then - echo "expecting tag set to be successful. exiting.." - exit_1; + echo "expecting tag set to be successful. exiting.." + exit_1 fi sleep 5 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) if [ "${val}" != "val1" ]; then - echo "expected bucket tag to have replicated, exiting..." - exit_1; + echo "expected bucket tag to have replicated, exiting..." + exit_1 fi # Create user with policy consoleAdmin on minio1 ./mc admin user add minio1 foobarx foobar123 if [ $? -ne 0 ]; then - echo "adding user failed, exiting.." - exit_1; + echo "adding user failed, exiting.." + exit_1 fi ./mc admin policy attach minio1 consoleAdmin --user=foobarx if [ $? -ne 0 ]; then - echo "adding policy mapping failed, exiting.." - exit_1; + echo "adding policy mapping failed, exiting.." + exit_1 fi sleep 10 # unset policy for foobarx in minio2 ./mc admin policy detach minio2 consoleAdmin --user=foobarx if [ $? -ne 0 ]; then - echo "unset policy mapping failed, exiting.." - exit_1; + echo "unset policy mapping failed, exiting.." + exit_1 fi # create a bucket bucket2 on minio1. @@ -340,8 +340,8 @@ sleep 10 # Test whether policy detach replicated to minio1 policy=$(./mc admin user info minio1 foobarx --json | jq -r .policyName) if [ "${policy}" != "null" ]; then - echo "expected policy detach to have replicated, exiting..." - exit_1; + echo "expected policy detach to have replicated, exiting..." + exit_1 fi kill -9 ${site1_pid} @@ -358,15 +358,15 @@ minio server --config-dir /tmp/minio-internal --address ":9001" /tmp/minio-inter sleep 200 # Test whether most recent tag update on minio2 is replicated to minio1 -val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key ) +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) if [ "${val}" != "val2" ]; then - echo "expected bucket tag to have replicated, exiting..." - exit_1; + echo "expected bucket tag to have replicated, exiting..." + exit_1 fi # Test if bucket created/deleted when minio1 is down healed -diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null -if [ $? -ne 0 ]; then - echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." - exit_1; +diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null +if [ $? -ne 0 ]; then + echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." + exit_1 fi diff --git a/docs/site-replication/run-multi-site-oidc.sh b/docs/site-replication/run-multi-site-oidc.sh index 97010cea4..27d6025b5 100755 --- a/docs/site-replication/run-multi-site-oidc.sh +++ b/docs/site-replication/run-multi-site-oidc.sh @@ -2,23 +2,23 @@ # shellcheck disable=SC2120 exit_1() { - cleanup + cleanup - echo "minio1 ============" - cat /tmp/minio1_1.log - echo "minio2 ============" - cat /tmp/minio2_1.log - echo "minio3 ============" - cat /tmp/minio3_1.log + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + echo "minio3 ============" + cat /tmp/minio3_1.log - exit 1 + exit 1 } cleanup() { - echo "Cleaning up instances of MinIO" - pkill minio - pkill -9 minio - rm -rf /tmp/minio{1,2,3} + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/minio{1,2,3} } cleanup @@ -53,8 +53,8 @@ minio server --address ":9003" --console-address ":12000" /tmp/minio3/{1...4} >/ site3_pid=$! if [ ! -f ./mc ]; then - wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \ - && chmod +x mc + wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc fi sleep 10 @@ -70,13 +70,13 @@ sleep 5 ./mc admin policy info minio2 projecta >/dev/null 2>&1 if [ $? -ne 0 ]; then - echo "expecting the command to succeed, exiting.." - exit_1; + echo "expecting the command to succeed, exiting.." + exit_1 fi ./mc admin policy info minio3 projecta >/dev/null 2>&1 if [ $? -ne 0 ]; then - echo "expecting the command to succeed, exiting.." - exit_1; + echo "expecting the command to succeed, exiting.." + exit_1 fi ./mc admin policy remove minio3 projecta @@ -84,14 +84,14 @@ fi sleep 10 ./mc admin policy info minio1 projecta if [ $? -eq 0 ]; then - echo "expecting the command to fail, exiting.." - exit_1; + echo "expecting the command to fail, exiting.." + exit_1 fi ./mc admin policy info minio2 projecta if [ $? -eq 0 ]; then - echo "expecting the command to fail, exiting.." - exit_1; + echo "expecting the command to fail, exiting.." + exit_1 fi ./mc admin policy create minio1 projecta ./docs/site-replication/rw.json @@ -102,8 +102,8 @@ STS_CRED=$(MINIO_ENDPOINT=http://localhost:9001 go run ./docs/site-replication/g MC_HOST_foo=http://${STS_CRED}@localhost:9001 ./mc ls foo if [ $? -ne 0 ]; then - echo "Expected sts credential to work, exiting.." - exit_1; + echo "Expected sts credential to work, exiting.." + exit_1 fi sleep 2 @@ -111,14 +111,14 @@ sleep 2 # Check that the STS credential works on minio2 and minio3. MC_HOST_foo=http://${STS_CRED}@localhost:9002 ./mc ls foo if [ $? -ne 0 ]; then - echo "Expected sts credential to work, exiting.." - exit_1; + echo "Expected sts credential to work, exiting.." + exit_1 fi MC_HOST_foo=http://${STS_CRED}@localhost:9003 ./mc ls foo if [ $? -ne 0 ]; then - echo "Expected sts credential to work, exiting.." - exit_1; + echo "Expected sts credential to work, exiting.." + exit_1 fi STS_ACCESS_KEY=$(echo ${STS_CRED} | cut -d ':' -f 1) @@ -126,41 +126,41 @@ STS_ACCESS_KEY=$(echo ${STS_CRED} | cut -d ':' -f 1) # Create service account for STS user ./mc admin user svcacct add minio2 $STS_ACCESS_KEY --access-key testsvc --secret-key testsvc123 if [ $? -ne 0 ]; then - echo "adding svc account failed, exiting.." - exit_1; + echo "adding svc account failed, exiting.." + exit_1 fi sleep 10 ./mc admin user svcacct info minio1 testsvc if [ $? -ne 0 ]; then - echo "svc account not mirrored, exiting.." - exit_1; + echo "svc account not mirrored, exiting.." + exit_1 fi ./mc admin user svcacct info minio2 testsvc if [ $? -ne 0 ]; then - echo "svc account not mirrored, exiting.." - exit_1; + echo "svc account not mirrored, exiting.." + exit_1 fi ./mc admin user svcacct rm minio1 testsvc if [ $? -ne 0 ]; then - echo "removing svc account failed, exiting.." - exit_1; + echo "removing svc account failed, exiting.." + exit_1 fi sleep 10 ./mc admin user svcacct info minio2 testsvc if [ $? -eq 0 ]; then - echo "svc account found after delete, exiting.." - exit_1; + echo "svc account found after delete, exiting.." + exit_1 fi ./mc admin user svcacct info minio3 testsvc if [ $? -eq 0 ]; then - echo "svc account found after delete, exiting.." - exit_1; + echo "svc account found after delete, exiting.." + exit_1 fi # create a bucket bucket2 on minio1. @@ -176,14 +176,14 @@ expected_checksum=$(cat ./lrgfile | md5sum) sleep 5 ./mc stat minio2/newbucket if [ $? -ne 0 ]; then - echo "expecting bucket to be present. exiting.." - exit_1; + echo "expecting bucket to be present. exiting.." + exit_1 fi ./mc stat minio3/newbucket if [ $? -ne 0 ]; then - echo "expecting bucket to be present. exiting.." - exit_1; + echo "expecting bucket to be present. exiting.." + exit_1 fi ./mc cp README.md minio2/newbucket/ @@ -191,14 +191,14 @@ fi sleep 5 ./mc stat minio1/newbucket/README.md if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi ./mc stat minio3/newbucket/README.md if [ $? -ne 0 ]; then - echo "expecting object to be present. exiting.." - exit_1; + echo "expecting object to be present. exiting.." + exit_1 fi ./mc rm minio3/newbucket/README.md @@ -206,66 +206,66 @@ sleep 5 ./mc stat minio2/newbucket/README.md if [ $? -eq 0 ]; then - echo "expected file to be deleted, exiting.." - exit_1; + echo "expected file to be deleted, exiting.." + exit_1 fi ./mc stat minio1/newbucket/README.md if [ $? -eq 0 ]; then - echo "expected file to be deleted, exiting.." - exit_1; + echo "expected file to be deleted, exiting.." + exit_1 fi sleep 10 ./mc stat minio3/newbucket/lrgfile if [ $? -ne 0 ]; then - echo "expected object to be present, exiting.." - exit_1; + echo "expected object to be present, exiting.." + exit_1 fi actual_checksum=$(./mc cat minio3/newbucket/lrgfile | md5sum) if [ "${expected_checksum}" != "${actual_checksum}" ]; then - echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" - exit + echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" + exit fi rm ./lrgfile ./mc mb --with-lock minio3/newbucket-olock sleep 5 -enabled_minio2=$(./mc stat --json minio2/newbucket-olock| jq -r .ObjectLock.enabled) +enabled_minio2=$(./mc stat --json minio2/newbucket-olock | jq -r .ObjectLock.enabled) if [ $? -ne 0 ]; then - echo "expected bucket to be mirrored with object-lock but not present, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 fi if [ "${enabled_minio2}" != "Enabled" ]; then - echo "expected bucket to be mirrored with object-lock enabled, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 fi -enabled_minio1=$(./mc stat --json minio1/newbucket-olock| jq -r .ObjectLock.enabled) +enabled_minio1=$(./mc stat --json minio1/newbucket-olock | jq -r .ObjectLock.enabled) if [ $? -ne 0 ]; then - echo "expected bucket to be mirrored with object-lock but not present, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 fi if [ "${enabled_minio1}" != "Enabled" ]; then - echo "expected bucket to be mirrored with object-lock enabled, exiting..." - exit_1; + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 fi # "Test if most recent tag update is replicated" ./mc tag set minio2/newbucket "key=val1" if [ $? -ne 0 ]; then - echo "expecting tag set to be successful. exiting.." - exit_1; + echo "expecting tag set to be successful. exiting.." + exit_1 fi sleep 10 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) if [ "${val}" != "val1" ]; then - echo "expected bucket tag to have replicated, exiting..." - exit_1; + echo "expected bucket tag to have replicated, exiting..." + exit_1 fi # stop minio1 instance kill -9 ${site1_pid} @@ -281,15 +281,15 @@ minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/ sleep 200 # Test whether most recent tag update on minio2 is replicated to minio1 -val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key ) +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) if [ "${val}" != "val2" ]; then - echo "expected bucket tag to have replicated, exiting..." - exit_1; + echo "expected bucket tag to have replicated, exiting..." + exit_1 fi # Test if bucket created/deleted when minio1 is down healed -diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null -if [ $? -ne 0 ]; then - echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." - exit_1; +diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null +if [ $? -ne 0 ]; then + echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." + exit_1 fi