re-implement pickValidInfo dataDir, move to quorum calculation (#13681)

dataDir loosely based on maxima is incorrect and does not work in all situations such as disks in the following order - xl.json migration to xl.meta there may be partial xl.json's leftover if some disks are not yet connected when the disk is yet to come up, since xl.json mtime and xl.meta is same the dataDir maxima doesn't work properly leading to quorum issues. - its also possible that XLV1 might be true among the disks available, make sure to keep FileInfo based on common quorum and skip unexpected disks with the older data format. Also, this PR tests upgrade from older to a newer release if the data is readable and matches the checksum. NOTE: this is just initial work we can build on top of this to do further tests.
2025-11-07 12:52:58 -05:00 · 2021-11-21 10:41:30 -08:00
parent 36b5426f6e
commit c791de0e1e
20 changed files with 378 additions and 155 deletions
--- a/buildscripts/minio-upgrade.sh
+++ b/buildscripts/minio-upgrade.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+trap 'cleanup $LINENO' ERR
+
+# shellcheck disable=SC2120
+cleanup() {
+    MINIO_VERSION=dev docker-compose \
+                 -f "buildscripts/upgrade-tests/compose.yml" \
+                 rm -s -f
+    docker volume prune -f
+}
+
+__init__() {
+    sudo apt install curl -y
+    export GOPATH=/tmp/gopath
+    export PATH=${PATH}:${GOPATH}/bin
+
+    go install github.com/minio/mc@latest
+
+    TAG=minio/minio:dev make docker
+
+    MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \
+                 -f "buildscripts/upgrade-tests/compose.yml" \
+                 up -d --build
+    until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do
+        echo "...waiting..." && sleep 5;
+    done
+
+    mc mb minio/minio-test/
+    mc cp ./minio minio/minio-test/to-read/
+    mc cp /etc/hosts minio/minio-test/to-read/hosts
+    mc policy set download minio/minio-test
+    mc cat minio/minio-test/to-read/minio | sha256sum
+    mc cat ./minio | sha256sum
+    curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum
+
+    MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
+}
+
+verify_checksum_after_heal() {
+    sum1=$(curl -s "$2" | sha256sum);
+    mc admin heal --json -r "$1" >/dev/null; # test after healing
+    sum1_heal=$(curl -s "$2" | sha256sum);
+
+    if [ "${sum1_heal}" != "${sum1}" ]; then
+        echo "mismatch expected ${sum1_heal}, got ${sum1}"
+        exit 1;
+    fi
+}
+
+verify_checksum_mc() {
+    expected=$(mc cat "$1" | sha256sum)
+    got=$(mc cat "$2" | sha256sum)
+
+    if [ "${expected}" != "${got}" ]; then
+        echo "mismatch expected ${expected}, got ${got}"
+        exit 1;
+    fi
+}
+
+main() {
+    MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
+
+    until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do
+        echo "...waiting..." && sleep 5
+    done
+
+    verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts
+
+    verify_checksum_mc ./minio minio/minio-test/to-read/minio
+
+    verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts
+
+    cleanup
+}
+
+( __init__ "$@" && main "$@" )
--- a/buildscripts/upgrade-tests/compose.yml
+++ b/buildscripts/upgrade-tests/compose.yml
@@ -0,0 +1,81 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${MINIO_VERSION}
+  command: server http://minio{1...4}/data{1...3}
+  env_file:
+    - ./minio.env
+  expose:
+    - "9000"
+    - "9001"
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - data1-1:/data1
+      - data1-2:/data2
+      - data1-3:/data3
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - data2-1:/data1
+      - data2-2:/data2
+      - data2-3:/data3
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - data3-1:/data1
+      - data3-2:/data2
+      - data3-3:/data3
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - data4-1:/data1
+      - data4-2:/data2
+      - data4-3:/data3
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    ports:
+    - "9000:9000"
+    - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  data1-1:
+  data1-2:
+  data1-3:
+  data2-1:
+  data2-2:
+  data2-3:
+  data3-1:
+  data3-2:
+  data3-3:
+  data4-1:
+  data4-2:
+  data4-3:
--- a/buildscripts/upgrade-tests/minio.env
+++ b/buildscripts/upgrade-tests/minio.env
@@ -0,0 +1,3 @@
+MINIO_ACCESS_KEY=minioadmin
+MINIO_SECRET_KEY=minioadmin
+MINIO_BROWSER=off
--- a/buildscripts/upgrade-tests/nginx.conf
+++ b/buildscripts/upgrade-tests/nginx.conf
@@ -0,0 +1,68 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    # main minio
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+         # To allow special characters in headers
+         ignore_invalid_headers off;
+         # Allow any size file to be uploaded.
+         # Set to a value such as 1000m; to restrict file size to a specific value
+         client_max_body_size 0;
+         # To disable buffering
+         proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}