From 6817c5ea58fbc7975e96d2af7801d358a7d4ede4 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 22 Apr 2020 16:06:58 -0700 Subject: [PATCH] migrate mint tests to latest versions (#9424) --- Dockerfile.mint | 10 +--- Dockerfile.simpleci | 4 +- buildscripts/gateway-tests.sh | 2 +- mint/Dockerfile.dev | 87 ------------------------------ mint/README.md | 2 +- mint/build/aws-sdk-ruby/install.sh | 4 +- mint/build/awscli/install.sh | 6 +-- mint/build/minio-dotnet/install.sh | 4 +- mint/build/minio-py/install.sh | 8 ++- mint/install-packages.list | 19 +++---- mint/mint.sh | 3 +- mint/preinstall.sh | 18 +++++-- mint/remove-packages.list | 2 +- mint/run/core/aws-sdk-go/run.sh | 2 +- mint/run/core/aws-sdk-java/run.sh | 4 +- mint/run/core/aws-sdk-php/run.sh | 2 +- mint/run/core/aws-sdk-ruby/run.sh | 2 +- mint/run/core/awscli/run.sh | 2 +- mint/run/core/healthcheck/run.sh | 2 +- mint/run/core/mc/run.sh | 2 +- mint/run/core/minio-dotnet/run.sh | 2 +- mint/run/core/minio-go/run.sh | 2 +- mint/run/core/minio-java/run.sh | 2 +- mint/run/core/minio-js/run.sh | 2 +- mint/run/core/minio-py/run.sh | 2 +- mint/run/core/s3cmd/run.sh | 2 +- mint/run/core/s3select/run.sh | 2 +- mint/run/core/security/run.sh | 2 +- mint/run/core/worm/run.sh | 2 +- 29 files changed, 57 insertions(+), 146 deletions(-) delete mode 100644 mint/Dockerfile.dev diff --git a/Dockerfile.mint b/Dockerfile.mint index 3a4d93ea4..6daffbaaf 100644 --- a/Dockerfile.mint +++ b/Dockerfile.mint @@ -1,17 +1,11 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 ENV DEBIAN_FRONTEND noninteractive - ENV LANG C.UTF-8 - ENV GOROOT /usr/local/go - -ENV GOPATH /usr/local - +ENV GOPATH /usr/local/gopath ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH - ENV MINT_ROOT_DIR /mint - COPY mint /mint RUN apt-get --yes update && apt-get --yes upgrade && \ diff --git a/Dockerfile.simpleci b/Dockerfile.simpleci index bff2f369b..c1a8245ce 100644 --- a/Dockerfile.simpleci +++ b/Dockerfile.simpleci @@ -57,7 +57,7 @@ RUN yarn test #------------------------------------------------------------- # Stage 3: Run Gateway Tests #------------------------------------------------------------- -FROM ubuntu:16.04 +FROM ubuntu:18.04 COPY --from=0 /go/src/github.com/minio/minio/minio /usr/bin/minio COPY buildscripts/gateway-tests.sh /usr/bin/gateway-tests.sh @@ -66,7 +66,7 @@ COPY mint /mint ENV DEBIAN_FRONTEND noninteractive ENV LANG C.UTF-8 ENV GOROOT /usr/local/go -ENV GOPATH /usr/local +ENV GOPATH /usr/local/gopath ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH ENV SIMPLE_CI 1 ENV MINT_ROOT_DIR /mint diff --git a/buildscripts/gateway-tests.sh b/buildscripts/gateway-tests.sh index 659e292d7..63e7dfaab 100755 --- a/buildscripts/gateway-tests.sh +++ b/buildscripts/gateway-tests.sh @@ -48,7 +48,7 @@ function main() SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \ SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \ aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \ - healthcheck mc minio-dotnet minio-java minio-js \ + healthcheck mc minio-dotnet minio-js \ minio-py s3cmd s3select security rv=$? diff --git a/mint/Dockerfile.dev b/mint/Dockerfile.dev deleted file mode 100644 index 9e4c30326..000000000 --- a/mint/Dockerfile.dev +++ /dev/null @@ -1,87 +0,0 @@ -FROM ubuntu:16.04 - -ENV DEBIAN_FRONTEND noninteractive - -ENV LANG C.UTF-8 - -ENV GOROOT /usr/local/go - -ENV GOPATH /usr/local - -ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH - -WORKDIR /mint - -RUN apt-get --yes update && apt-get --yes upgrade && \ - apt-get --yes --quiet install wget jq curl dnsmasq - -ENV MINT_ROOT_DIR /mint -ENV MINT_RUN_CORE_DIR $MINT_ROOT_DIR/run/core -ENV MINT_RUN_SECURITY_DIR $MINT_ROOT_DIR/run/security -ENV WGET "wget --quiet --no-check-certificate" - -COPY create-data-files.sh /mint -RUN /mint/create-data-files.sh - -COPY install-packages.list /mint -COPY preinstall.sh /mint -RUN /mint/preinstall.sh - -COPY run /mint/run - -COPY build/awscli /mint/build/awscli -RUN build/awscli/install.sh - -COPY build/aws-sdk-java /mint/build/aws-sdk-java -RUN build/aws-sdk-java/install.sh - -COPY build/aws-sdk-go /mint/build/aws-sdk-go -RUN build/aws-sdk-go/install.sh - -COPY build/aws-sdk-php /mint/build/aws-sdk-php -RUN build/aws-sdk-php/install.sh - -COPY build/aws-sdk-ruby /mint/build/aws-sdk-ruby -RUN build/aws-sdk-ruby/install.sh - -COPY build/mc /mint/build/mc -RUN build/mc/install.sh - -COPY build/minio-go /mint/build/minio-go -RUN build/minio-go/install.sh - -COPY build/minio-java /mint/build/minio-java -RUN build/minio-java/install.sh - -COPY build/minio-js /mint/build/minio-js -RUN build/minio-js/install.sh - -COPY build/minio-py /mint/build/minio-py -RUN build/minio-py/install.sh - -COPY build/s3cmd /mint/build/s3cmd -RUN build/s3cmd/install.sh - -COPY build/minio-dotnet/ /mint/build/minio-dotnet/ -RUN /mint/build/minio-dotnet/install.sh - -COPY build/security /mint/build/security -RUN build/security/install.sh - -COPY build/worm /mint/build/worm -RUN build/worm/install.sh - -COPY build/healthcheck /mint/build/healthcheck -RUN build/healthcheck/install.sh - -COPY build/s3select /mint/build/s3select -RUN build/s3select/install.sh - -COPY remove-packages.list /mint -COPY postinstall.sh /mint -RUN /mint/postinstall.sh - -COPY mint.sh /mint/mint.sh -COPY entrypoint.sh /mint/entrypoint.sh - -ENTRYPOINT ["/mint/entrypoint.sh"] diff --git a/mint/README.md b/mint/README.md index 6f3d1a11c..580080856 100644 --- a/mint/README.md +++ b/mint/README.md @@ -82,7 +82,7 @@ All test logs are stored in `/mint/log/log.json` as multiple JSON document. Bel After making changes to Mint source code a local docker image can be built/run by ```sh -$ docker build -t minio/mint . -f Dockerfile.dev +$ docker build -t minio/mint . -f Dockerfile.mint $ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \ -e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest diff --git a/mint/build/aws-sdk-ruby/install.sh b/mint/build/aws-sdk-ruby/install.sh index a7d36555c..7c7055f5a 100755 --- a/mint/build/aws-sdk-ruby/install.sh +++ b/mint/build/aws-sdk-ruby/install.sh @@ -1,6 +1,6 @@ #!/bin/bash -e # -# Mint (C) 2017 Minio, Inc. +# Mint (C) 20172-2020 Minio, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,4 +15,4 @@ # limitations under the License. # -gem install --no-rdoc --no-ri aws-sdk-resources:3.56.0 aws-sdk multipart_body +gem install --no-rdoc --no-ri aws-sdk-resources aws-sdk multipart_body diff --git a/mint/build/awscli/install.sh b/mint/build/awscli/install.sh index 3dc98a5f1..cf7d9bd6a 100755 --- a/mint/build/awscli/install.sh +++ b/mint/build/awscli/install.sh @@ -1,6 +1,6 @@ #!/bin/bash -e # -# Mint (C) 2017 Minio, Inc. +# Mint (C) 2017-2020 Minio, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,4 @@ # limitations under the License. # -AWS_CLI_VERSION="1.16.309" - -python -m pip install awscli==$AWS_CLI_VERSION +pip3 install awscli --upgrade diff --git a/mint/build/minio-dotnet/install.sh b/mint/build/minio-dotnet/install.sh index 240fe559a..fc00e8592 100755 --- a/mint/build/minio-dotnet/install.sh +++ b/mint/build/minio-dotnet/install.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Mint (C) 2017 Minio, Inc. +# Mint (C) 2017-2020 Minio, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -39,4 +39,4 @@ rm -fr "${temp_dir}" cd "$MINIO_DOTNET_SDK_PATH" dotnet restore /p:Configuration=Mint -dotnet publish --runtime ubuntu.16.04-x64 --output out /p:Configuration=Mint +dotnet publish --runtime ubuntu.18.04-x64 --output out /p:Configuration=Mint diff --git a/mint/build/minio-py/install.sh b/mint/build/minio-py/install.sh index 8af9f98ef..39bd990e7 100755 --- a/mint/build/minio-py/install.sh +++ b/mint/build/minio-py/install.sh @@ -1,6 +1,6 @@ #!/bin/bash -e # -# Mint (C) 2017 Minio, Inc. +# Mint (C) 2017-2020 Minio, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,8 +22,6 @@ if [ -z "$MINIO_PY_VERSION" ]; then fi test_run_dir="$MINT_RUN_CORE_DIR/minio-py" -# FIX https://github.com/pypa/pip/issues/5221 -python -m pip install --upgrade pip -python -m pip install --user faker -python -m pip install minio=="$MINIO_PY_VERSION" +pip3 install --user faker +pip3 install minio=="$MINIO_PY_VERSION" $WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/${MINIO_PY_VERSION}/tests/functional/tests.py" diff --git a/mint/install-packages.list b/mint/install-packages.list index bc89dbbf7..600e36faf 100644 --- a/mint/install-packages.list +++ b/mint/install-packages.list @@ -1,17 +1,18 @@ git python3-pip nodejs -ruby -ruby-dev -ruby-bundler -php -php7.0-curl -php-xml -default-jre -default-jdk -ant +openjdk-8-jdk +openjdk-8-jre dirmngr +apt-transport-https dotnet-sdk-2.1 ca-certificates-mono nuget libunwind8 +ruby +ruby-dev +ruby-bundler +php +php-curl +php-xml +ant diff --git a/mint/mint.sh b/mint/mint.sh index e05c207b4..15d5684a3 100755 --- a/mint/mint.sh +++ b/mint/mint.sh @@ -111,7 +111,7 @@ function trust_s3_endpoint_tls_cert() # Download the public certificate from the server openssl s_client -showcerts -connect "$SERVER_ENDPOINT" /dev/null | \ openssl x509 -outform PEM -out /usr/local/share/ca-certificates/s3_server_cert.crt || \ - exit -1 + exit 1 # Load the certificate in the system update-ca-certificates --fresh >/dev/null @@ -191,7 +191,6 @@ function main() echo -n "($i/$count) Running $sdk_name tests ... " if ! run_test "$sdk_dir"; then (( i-- )) - break fi done diff --git a/mint/preinstall.sh b/mint/preinstall.sh index ee7ce368b..44d7a0b8f 100755 --- a/mint/preinstall.sh +++ b/mint/preinstall.sh @@ -1,6 +1,6 @@ #!/bin/bash -e # -# Mint (C) 2017 Minio, Inc. +# Mint (C) 2017-2020 Minio, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,13 +26,19 @@ fi $APT install apt-transport-https -wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb +if ! $WGET --output-document=packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb | bash -; then + echo "unable to download dotnet packages" + exit 1 +fi + dpkg -i packages-microsoft-prod.deb rm -f packages-microsoft-prod.deb + $APT update +$APT install gnupg ca-certificates # download and install golang -GO_VERSION="1.13.5" +GO_VERSION="1.13.10" GO_INSTALL_PATH="/usr/local" download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then @@ -42,7 +48,9 @@ fi xargs --arg-file="${MINT_ROOT_DIR}/install-packages.list" apt --quiet --yes install -# set python 3.5 as default -update-alternatives --install /usr/bin/python python /usr/bin/python3.5 1 +nuget update -self + +# set python 3.6 as default +update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 sync diff --git a/mint/remove-packages.list b/mint/remove-packages.list index a6c9ab148..0206fbc6d 100644 --- a/mint/remove-packages.list +++ b/mint/remove-packages.list @@ -3,7 +3,7 @@ git python3-pip ruby-dev ruby-bundler -default-jdk +openjdk-8-jdk ant dotnet nuget diff --git a/mint/run/core/aws-sdk-go/run.sh b/mint/run/core/aws-sdk-go/run.sh index 1811bdd53..813b3ee6a 100755 --- a/mint/run/core/aws-sdk-go/run.sh +++ b/mint/run/core/aws-sdk-go/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/aws-sdk-java/run.sh b/mint/run/core/aws-sdk-java/run.sh index 30c1e5c68..9a9ad38b0 100755 --- a/mint/run/core/aws-sdk-java/run.sh +++ b/mint/run/core/aws-sdk-java/run.sh @@ -18,13 +18,13 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" error_log_file="$2" # run tests -cd /mint/run/core/aws-sdk-java/ || exit -1 +cd /mint/run/core/aws-sdk-java/ || exit 1 java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-php/run.sh b/mint/run/core/aws-sdk-php/run.sh index 3a5c0b890..e3a37a37c 100755 --- a/mint/run/core/aws-sdk-php/run.sh +++ b/mint/run/core/aws-sdk-php/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/aws-sdk-ruby/run.sh b/mint/run/core/aws-sdk-ruby/run.sh index 73e647b79..1e59a71c7 100755 --- a/mint/run/core/aws-sdk-ruby/run.sh +++ b/mint/run/core/aws-sdk-ruby/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/awscli/run.sh b/mint/run/core/awscli/run.sh index 7ce73ee17..98bd61b19 100755 --- a/mint/run/core/awscli/run.sh +++ b/mint/run/core/awscli/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/healthcheck/run.sh b/mint/run/core/healthcheck/run.sh index 96e64abe4..727c5a856 100755 --- a/mint/run/core/healthcheck/run.sh +++ b/mint/run/core/healthcheck/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/mc/run.sh b/mint/run/core/mc/run.sh index 2bdc67d4d..b13c5e296 100755 --- a/mint/run/core/mc/run.sh +++ b/mint/run/core/mc/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/minio-dotnet/run.sh b/mint/run/core/minio-dotnet/run.sh index f5fc00ad8..473a8e141 100755 --- a/mint/run/core/minio-dotnet/run.sh +++ b/mint/run/core/minio-dotnet/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/minio-go/run.sh b/mint/run/core/minio-go/run.sh index 5ffa4b86f..5a3255bc6 100755 --- a/mint/run/core/minio-go/run.sh +++ b/mint/run/core/minio-go/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/minio-java/run.sh b/mint/run/core/minio-java/run.sh index 8ed724a4a..6a2fc14ae 100755 --- a/mint/run/core/minio-java/run.sh +++ b/mint/run/core/minio-java/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/minio-js/run.sh b/mint/run/core/minio-js/run.sh index 99b12b7f3..c6a0c58a9 100755 --- a/mint/run/core/minio-js/run.sh +++ b/mint/run/core/minio-js/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/minio-py/run.sh b/mint/run/core/minio-py/run.sh index 7d15357b1..2538783c0 100755 --- a/mint/run/core/minio-py/run.sh +++ b/mint/run/core/minio-py/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/s3cmd/run.sh b/mint/run/core/s3cmd/run.sh index 2d9bca656..e6cdc3fc3 100755 --- a/mint/run/core/s3cmd/run.sh +++ b/mint/run/core/s3cmd/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/s3select/run.sh b/mint/run/core/s3select/run.sh index 46db553e9..0574ea85f 100755 --- a/mint/run/core/s3select/run.sh +++ b/mint/run/core/s3select/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/security/run.sh b/mint/run/core/security/run.sh index 9283c76ee..9e7c2175a 100755 --- a/mint/run/core/security/run.sh +++ b/mint/run/core/security/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1" diff --git a/mint/run/core/worm/run.sh b/mint/run/core/worm/run.sh index 9404b8641..e529de870 100755 --- a/mint/run/core/worm/run.sh +++ b/mint/run/core/worm/run.sh @@ -18,7 +18,7 @@ # handle command line arguments if [ $# -ne 2 ]; then echo "usage: run.sh " - exit -1 + exit 1 fi output_log_file="$1"