diff --git a/.github/workflows/go-healing.yml b/.github/workflows/go-healing.yml index 7453e98b2..9dbfaaa3d 100644 --- a/.github/workflows/go-healing.yml +++ b/.github/workflows/go-healing.yml @@ -48,3 +48,4 @@ jobs: sudo sysctl net.ipv6.conf.default.disable_ipv6=0 make verify-healing make verify-healing-inconsistent-versions + make verify-healing-with-root-disks diff --git a/Makefile b/Makefile index 7d906b667..e9864791b 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,11 @@ verify-healing: ## verify healing and replacing disks with minio binary @(env bash $(PWD)/buildscripts/verify-healing.sh) @(env bash $(PWD)/buildscripts/unaligned-healing.sh) +verify-healing-with-root-disks: + @echo "Verify healing with root disks" + @GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null + @(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh) + verify-healing-inconsistent-versions: ## verify resolving inconsistent versions @echo "Verify resolving inconsistent versions build with race" @GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null diff --git a/buildscripts/verify-healing-with-root-disks.sh b/buildscripts/verify-healing-with-root-disks.sh new file mode 100755 index 000000000..831943b3d --- /dev/null +++ b/buildscripts/verify-healing-with-root-disks.sh @@ -0,0 +1,97 @@ +#!/bin/bash -e + +set -E +set -o pipefail +set -x + + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +WORK_DIR="$(mktemp -d)" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server ) + + +function start_minio() { + start_port=$1 + + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + unset MINIO_CI_CD + unset CI + + args=() + for i in $(seq 1 4); do + args+=("http://localhost:$[${start_port}+$i]${WORK_DIR}/mnt/disk$i/ ") + done + + for i in $(seq 1 4); do + "${MINIO[@]}" --address ":$[$start_port+$i]" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" & + done + + # Wait until all nodes return 403 + for i in $(seq 1 4); do + while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$[$start_port+$i])" -ne "403" ]; do + echo -n "."; + sleep 1; + done + done + + } + +# Prepare fake disks with losetup +function prepare_block_devices() { + mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/ + for i in 1 2 3 4; do + dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048 + mkfs.ext4 -F ${WORK_DIR}/disks/img.$i + sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i] + sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i + mkdir -p ${WORK_DIR}/mnt/disk$i/ + sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/ + sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/ + done +} + +# Start a distributed MinIO setup, unmount one disk and check if it is formatted +function main() { + start_port=$(shuf -i 10000-65000 -n 1) + start_minio ${start_port} + + # Unmount the disk, after the unmount the device id + # /tmp/xxx/mnt/disk4 will be the same as '/' and it + # will be detected as root disk + while [ "$u" != "0" ]; do + sudo umount ${WORK_DIR}/mnt/disk4/ + u=$? + sleep 1 + done + + # Wait until MinIO self heal kicks in + sleep 60 + + if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then + echo "A root disk is formatted unexpectedely" + cat "${WORK_DIR}/server4.log" + exit -1 + fi +} + +function cleanup() { + pkill minio + sudo umount ${WORK_DIR}/mnt/disk{1..3}/ + sudo rm /dev/minio-loopdisk* + rm -rf "$WORK_DIR" +} + +( prepare_block_devices ) +( main "$@" ) +rv=$? + +cleanup +exit "$rv" +