From c50627ee3e2df55bd41c68c3d7177991942547b0 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 30 Sep 2023 03:13:56 -0700 Subject: [PATCH] Add tests for multipart upload overwrites on versioned buckets (#18142) --- .github/workflows/mint.yml | 15 ++- .../mint/minio-compress-encrypt.yaml | 2 +- .github/workflows/mint/minio-erasure.yaml | 2 +- .github/workflows/mint/minio-pools.yaml | 2 +- .../multipart/docker-compose-site1.yaml | 66 +++++++++++ .../multipart/docker-compose-site2.yaml | 66 +++++++++++ .github/workflows/multipart/migrate.sh | 111 ++++++++++++++++++ .github/workflows/multipart/nginx-site1.conf | 61 ++++++++++ .github/workflows/multipart/nginx-site2.conf | 61 ++++++++++ Makefile | 2 +- 10 files changed, 382 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/multipart/docker-compose-site1.yaml create mode 100644 .github/workflows/multipart/docker-compose-site2.yaml create mode 100755 .github/workflows/multipart/migrate.sh create mode 100644 .github/workflows/multipart/nginx-site1.conf create mode 100644 .github/workflows/multipart/nginx-site2.conf diff --git a/.github/workflows/mint.yml b/.github/workflows/mint.yml index ffd142f1a..93ccb5992 100644 --- a/.github/workflows/mint.yml +++ b/.github/workflows/mint.yml @@ -37,7 +37,11 @@ jobs: - name: build-minio run: | - TAG="minio/minio:${{ steps.vars.outputs.sha_short }}" make docker + TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker + + - name: multipart uploads test + run: | + ${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}" - name: compress and encrypt run: | @@ -59,7 +63,14 @@ jobs: docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true done - docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }} + + docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true + docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true + for volume in $(docker volume ls -q | grep minio); do + docker volume rm ${volume} || true + done + + docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }} docker system prune -f || true docker volume prune -f || true docker volume rm $(docker volume ls -q -f dangling=true) || true diff --git a/.github/workflows/mint/minio-compress-encrypt.yaml b/.github/workflows/mint/minio-compress-encrypt.yaml index 9176df5a7..0ebaacb99 100644 --- a/.github/workflows/mint/minio-compress-encrypt.yaml +++ b/.github/workflows/mint/minio-compress-encrypt.yaml @@ -2,7 +2,7 @@ version: '3.7' # Settings and configurations that are common for all containers x-minio-common: &minio-common - image: minio/minio:${JOB_NAME} + image: quay.io/minio/minio:${JOB_NAME} command: server --console-address ":9001" http://minio{1...4}/cdata{1...2} expose: - "9000" diff --git a/.github/workflows/mint/minio-erasure.yaml b/.github/workflows/mint/minio-erasure.yaml index 725eb20b1..e28c137d6 100644 --- a/.github/workflows/mint/minio-erasure.yaml +++ b/.github/workflows/mint/minio-erasure.yaml @@ -2,7 +2,7 @@ version: '3.7' # Settings and configurations that are common for all containers x-minio-common: &minio-common - image: minio/minio:${JOB_NAME} + image: quay.io/minio/minio:${JOB_NAME} command: server --console-address ":9001" edata{1...4} expose: - "9000" diff --git a/.github/workflows/mint/minio-pools.yaml b/.github/workflows/mint/minio-pools.yaml index 88ac8a0aa..0a54ca353 100644 --- a/.github/workflows/mint/minio-pools.yaml +++ b/.github/workflows/mint/minio-pools.yaml @@ -2,7 +2,7 @@ version: '3.7' # Settings and configurations that are common for all containers x-minio-common: &minio-common - image: minio/minio:${JOB_NAME} + image: quay.io/minio/minio:${JOB_NAME} command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2} expose: - "9000" diff --git a/.github/workflows/multipart/docker-compose-site1.yaml b/.github/workflows/multipart/docker-compose-site1.yaml new file mode 100644 index 000000000..b4a665ef0 --- /dev/null +++ b/.github/workflows/multipart/docker-compose-site1.yaml @@ -0,0 +1,66 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:${RELEASE} + command: server http://site1-minio{1...4}/data{1...2} + environment: + - MINIO_PROMETHEUS_AUTH_TYPE=public + - CI=true + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + site1-minio1: + <<: *minio-common + hostname: site1-minio1 + volumes: + - site1-data1-1:/data1 + - site1-data1-2:/data2 + + site1-minio2: + <<: *minio-common + hostname: site1-minio2 + volumes: + - site1-data2-1:/data1 + - site1-data2-2:/data2 + + site1-minio3: + <<: *minio-common + hostname: site1-minio3 + volumes: + - site1-data3-1:/data1 + - site1-data3-2:/data2 + + site1-minio4: + <<: *minio-common + hostname: site1-minio4 + volumes: + - site1-data4-1:/data1 + - site1-data4-2:/data2 + + site1-nginx: + image: nginx:1.19.2-alpine + hostname: site1-nginx + volumes: + - ./nginx-site1.conf:/etc/nginx/nginx.conf:ro + ports: + - "9001:9001" + depends_on: + - site1-minio1 + - site1-minio2 + - site1-minio3 + - site1-minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + site1-data1-1: + site1-data1-2: + site1-data2-1: + site1-data2-2: + site1-data3-1: + site1-data3-2: + site1-data4-1: + site1-data4-2: diff --git a/.github/workflows/multipart/docker-compose-site2.yaml b/.github/workflows/multipart/docker-compose-site2.yaml new file mode 100644 index 000000000..31a1f7a8c --- /dev/null +++ b/.github/workflows/multipart/docker-compose-site2.yaml @@ -0,0 +1,66 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:${RELEASE} + command: server http://site2-minio{1...4}/data{1...2} + environment: + - MINIO_PROMETHEUS_AUTH_TYPE=public + - CI=true + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + site2-minio1: + <<: *minio-common + hostname: site2-minio1 + volumes: + - site2-data1-1:/data1 + - site2-data1-2:/data2 + + site2-minio2: + <<: *minio-common + hostname: site2-minio2 + volumes: + - site2-data2-1:/data1 + - site2-data2-2:/data2 + + site2-minio3: + <<: *minio-common + hostname: site2-minio3 + volumes: + - site2-data3-1:/data1 + - site2-data3-2:/data2 + + site2-minio4: + <<: *minio-common + hostname: site2-minio4 + volumes: + - site2-data4-1:/data1 + - site2-data4-2:/data2 + + site2-nginx: + image: nginx:1.19.2-alpine + hostname: site2-nginx + volumes: + - ./nginx-site2.conf:/etc/nginx/nginx.conf:ro + ports: + - "9002:9002" + depends_on: + - site2-minio1 + - site2-minio2 + - site2-minio3 + - site2-minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + site2-data1-1: + site2-data1-2: + site2-data2-1: + site2-data2-2: + site2-data3-1: + site2-data3-2: + site2-data4-1: + site2-data4-2: diff --git a/.github/workflows/multipart/migrate.sh b/.github/workflows/multipart/migrate.sh new file mode 100755 index 000000000..77957d9d4 --- /dev/null +++ b/.github/workflows/multipart/migrate.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +set -x + +## change working directory +cd .github/workflows/multipart/ + +docker-compose -f docker-compose-site1.yaml rm -s -f +docker-compose -f docker-compose-site2.yaml rm -s -f +for volume in $(docker volume ls -q | grep minio); do + docker volume rm ${volume} +done + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +( + cd /tmp + go install github.com/minio/minio/docs/debugging/s3-check-md5@latest +) + +export RELEASE=RELEASE.2023-08-29T23-07-35Z + +docker-compose -f docker-compose-site1.yaml up -d +docker-compose -f docker-compose-site2.yaml up -d + +sleep 30 + +mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4 +mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4 + +./mc ready site1/ +./mc ready site2/ + +./mc admin replicate add site1 site2 +./mc mb site1/testbucket/ +./mc cp -r --quiet /usr/bin site1/testbucket/ + +sleep 5 + +s3-check-md5 -h + +failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l) +failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l) + +if [ $failed_count_site1 -ne 0 ]; then + echo "failed with multipart on site1 uploads" + exit 1 +fi + +if [ $failed_count_site2 -ne 0 ]; then + echo "failed with multipart on site2 uploads" + exit 1 +fi + +./mc cp -r --quiet /usr/bin site1/testbucket/ + +sleep 5 + +failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l) +failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l) + +## we do not need to fail here, since we are going to test +## upgrading to master, healing and being able to recover +## the last version. +if [ $failed_count_site1 -ne 0 ]; then + echo "failed with multipart on site1 uploads ${failed_count_site1}" +fi + +if [ $failed_count_site2 -ne 0 ]; then + echo "failed with multipart on site2 uploads ${failed_count_site2}" +fi + +export RELEASE=${1} + +docker-compose -f docker-compose-site1.yaml up -d +docker-compose -f docker-compose-site2.yaml up -d + +./mc ready site1/ +./mc ready site2/ + +./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null +./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null + +failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l) +failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l) + +if [ $failed_count_site1 -ne 0 ]; then + echo "failed with multipart on site1 uploads" + exit 1 +fi + +if [ $failed_count_site2 -ne 0 ]; then + echo "failed with multipart on site2 uploads" + exit 1 +fi + +docker-compose -f docker-compose-site1.yaml rm -s -f +docker-compose -f docker-compose-site2.yaml rm -s -f +for volume in $(docker volume ls -q | grep minio); do + docker volume rm ${volume} +done + +docker system prune -f || true +docker volume prune -f || true +docker volume rm $(docker volume ls -q -f dangling=true) || true + +## change working directory +cd ../../../ diff --git a/.github/workflows/multipart/nginx-site1.conf b/.github/workflows/multipart/nginx-site1.conf new file mode 100644 index 000000000..1a37b9d2b --- /dev/null +++ b/.github/workflows/multipart/nginx-site1.conf @@ -0,0 +1,61 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server site1-minio1:9000; + server site1-minio2:9000; + server site1-minio3:9000; + server site1-minio4:9000; + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } +} diff --git a/.github/workflows/multipart/nginx-site2.conf b/.github/workflows/multipart/nginx-site2.conf new file mode 100644 index 000000000..b93bba24d --- /dev/null +++ b/.github/workflows/multipart/nginx-site2.conf @@ -0,0 +1,61 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server site2-minio1:9000; + server site2-minio2:9000; + server site2-minio3:9000; + server site2-minio4:9000; + } + + server { + listen 9002; + listen [::]:9002; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } +} diff --git a/Makefile b/Makefile index adee4ccfe..bbe0cd0f7 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GOARCH := $(shell go env GOARCH) GOOS := $(shell go env GOOS) VERSION ?= $(shell git describe --tags) -TAG ?= "minio/minio:$(VERSION)" +TAG ?= "quay.io/minio/minio:$(VERSION)" GOLANGCI_VERSION = v1.51.2 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)