mirror of
https://github.com/minio/minio.git
synced 2025-01-11 15:03:22 -05:00
Add mint tests into MinIO repo (#7886)
This commit is contained in:
parent
2d96745156
commit
1dc25bcf5f
5
.gitignore
vendored
5
.gitignore
vendored
@ -9,7 +9,6 @@ site/
|
||||
/.idea/
|
||||
/Minio.iml
|
||||
**/access.log
|
||||
build/
|
||||
vendor/**/*.js
|
||||
vendor/**/*.json
|
||||
release
|
||||
@ -23,5 +22,5 @@ prime/
|
||||
stage/
|
||||
.sia_temp/
|
||||
config.json
|
||||
healthcheck
|
||||
check-user
|
||||
dockerscripts/check-user
|
||||
dockerscripts/healthcheck
|
14
.travis.yml
14
.travis.yml
@ -2,6 +2,14 @@ go_import_path: github.com/minio/minio
|
||||
|
||||
language: go
|
||||
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- shellcheck
|
||||
|
||||
services:
|
||||
- docker
|
||||
|
||||
# this ensures PRs based on a local branch are not built twice
|
||||
# the downside is that a PR targeting a different branch is not built
|
||||
# but as a workaround you can add the branch to this list
|
||||
@ -26,7 +34,7 @@ matrix:
|
||||
- $HOME/gopath/pkg/mod
|
||||
- $HOME/go/pkg/mod
|
||||
|
||||
go: 1.12.1
|
||||
go: 1.12.5
|
||||
script:
|
||||
- make
|
||||
- diff -au <(gofmt -s -d cmd) <(printf "")
|
||||
@ -37,12 +45,14 @@ matrix:
|
||||
- make verify
|
||||
- make coverage
|
||||
- cd browser && yarn && yarn test && cd ..
|
||||
- bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
|
||||
|
||||
- os: windows
|
||||
env:
|
||||
- ARCH=x86_64
|
||||
- CGO_ENABLED=0
|
||||
- GO111MODULE=on
|
||||
go: 1.12.1
|
||||
go: 1.12.5
|
||||
script:
|
||||
- go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
|
||||
- bash buildscripts/go-coverage.sh
|
||||
|
@ -33,7 +33,7 @@ USER ci
|
||||
RUN make
|
||||
RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
|
||||
RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
|
||||
RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
|
||||
RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
|
||||
RUN make verifiers
|
||||
RUN make crosscompile
|
||||
RUN make verify
|
||||
@ -56,15 +56,17 @@ FROM ubuntu:16.04
|
||||
|
||||
COPY --from=0 /go/src/github.com/minio/minio/minio /usr/bin/minio
|
||||
COPY buildscripts/gateway-tests.sh /usr/bin/gateway-tests.sh
|
||||
COPY mint /mint
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV LANG C.UTF-8
|
||||
ENV GOROOT /usr/local/go
|
||||
ENV GOPATH /usr/local
|
||||
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
|
||||
ENV MINT_ROOT_DIR /mint
|
||||
|
||||
RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl git dnsmasq && \
|
||||
git clone https://github.com/minio/mint.git /mint && \
|
||||
RUN apt-get --yes update && apt-get --yes upgrade && \
|
||||
apt-get --yes --quiet install wget jq curl git dnsmasq && \
|
||||
cd /mint && /mint/release.sh
|
||||
|
||||
WORKDIR /mint
|
||||
|
17
mint/.gitignore
vendored
Normal file
17
mint/.gitignore
vendored
Normal file
@ -0,0 +1,17 @@
|
||||
*.test
|
||||
*.jar
|
||||
src/*
|
||||
temp
|
||||
__pycache__/
|
||||
log/*
|
||||
minio.test
|
||||
bin/*
|
||||
node_modules
|
||||
# exception to the rule
|
||||
!log/.gitkeep
|
||||
!bin/.gitkeep
|
||||
*.class
|
||||
*~
|
||||
run/core/minio-dotnet/bin/*
|
||||
run/core/minio-dotnet/obj/*
|
||||
run/core/minio-dotnet/out/*
|
20
mint/Dockerfile
Normal file
20
mint/Dockerfile
Normal file
@ -0,0 +1,20 @@
|
||||
FROM ubuntu:16.04
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
ENV LANG C.UTF-8
|
||||
|
||||
ENV GOROOT /usr/local/go
|
||||
|
||||
ENV GOPATH /usr/local
|
||||
|
||||
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
|
||||
|
||||
RUN apt-get --yes update && apt-get --yes upgrade && \
|
||||
apt-get --yes --quiet install wget jq curl git dnsmasq && \
|
||||
git clone https://github.com/minio/minio.git /minio && \
|
||||
ln -sf /minio/mint /mint && /mint/release.sh
|
||||
|
||||
WORKDIR /mint
|
||||
|
||||
ENTRYPOINT ["/mint/entrypoint.sh"]
|
84
mint/Dockerfile.dev
Normal file
84
mint/Dockerfile.dev
Normal file
@ -0,0 +1,84 @@
|
||||
FROM ubuntu:16.04
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
ENV LANG C.UTF-8
|
||||
|
||||
ENV GOROOT /usr/local/go
|
||||
|
||||
ENV GOPATH /usr/local
|
||||
|
||||
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
|
||||
|
||||
WORKDIR /mint
|
||||
|
||||
RUN apt-get --yes update && apt-get --yes upgrade && \
|
||||
apt-get --yes --quiet install wget jq curl dnsmasq
|
||||
|
||||
ENV MINT_ROOT_DIR /mint
|
||||
ENV MINT_RUN_CORE_DIR $MINT_ROOT_DIR/run/core
|
||||
ENV MINT_RUN_SECURITY_DIR $MINT_ROOT_DIR/run/security
|
||||
ENV WGET "wget --quiet --no-check-certificate"
|
||||
|
||||
COPY create-data-files.sh /mint
|
||||
RUN /mint/create-data-files.sh
|
||||
|
||||
COPY install-packages.list /mint
|
||||
COPY preinstall.sh /mint
|
||||
RUN /mint/preinstall.sh
|
||||
|
||||
COPY run /mint/run
|
||||
|
||||
COPY build/awscli /mint/build/awscli
|
||||
RUN build/awscli/install.sh
|
||||
|
||||
COPY build/aws-sdk-java /mint/build/aws-sdk-java
|
||||
RUN build/aws-sdk-java/install.sh
|
||||
|
||||
COPY build/aws-sdk-go /mint/build/aws-sdk-go
|
||||
RUN build/aws-sdk-go/install.sh
|
||||
|
||||
COPY build/aws-sdk-php /mint/build/aws-sdk-php
|
||||
RUN build/aws-sdk-php/install.sh
|
||||
|
||||
COPY build/aws-sdk-ruby /mint/build/aws-sdk-ruby
|
||||
RUN build/aws-sdk-ruby/install.sh
|
||||
|
||||
COPY build/mc /mint/build/mc
|
||||
RUN build/mc/install.sh
|
||||
|
||||
COPY build/minio-go /mint/build/minio-go
|
||||
RUN build/minio-go/install.sh
|
||||
|
||||
COPY build/minio-java /mint/build/minio-java
|
||||
RUN build/minio-java/install.sh
|
||||
|
||||
COPY build/minio-js /mint/build/minio-js
|
||||
RUN build/minio-js/install.sh
|
||||
|
||||
COPY build/minio-py /mint/build/minio-py
|
||||
RUN build/minio-py/install.sh
|
||||
|
||||
COPY build/s3cmd /mint/build/s3cmd
|
||||
RUN build/s3cmd/install.sh
|
||||
|
||||
COPY build/minio-dotnet/ /mint/build/minio-dotnet/
|
||||
RUN /mint/build/minio-dotnet/install.sh
|
||||
|
||||
COPY build/security /mint/build/security
|
||||
RUN build/security/install.sh
|
||||
|
||||
COPY build/worm /mint/build/worm
|
||||
RUN build/worm/install.sh
|
||||
|
||||
COPY build/healthcheck /mint/build/healthcheck
|
||||
RUN build/healthcheck/install.sh
|
||||
|
||||
COPY remove-packages.list /mint
|
||||
COPY postinstall.sh /mint
|
||||
RUN /mint/postinstall.sh
|
||||
|
||||
COPY mint.sh /mint/mint.sh
|
||||
COPY entrypoint.sh /mint/entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/mint/entrypoint.sh"]
|
123
mint/README.md
Normal file
123
mint/README.md
Normal file
@ -0,0 +1,123 @@
|
||||
# Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/)
|
||||
|
||||
Mint is a testing framework for Minio object server, available as a docker image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests.
|
||||
|
||||
- awscli
|
||||
- aws-sdk-go
|
||||
- aws-sdk-php
|
||||
- aws-sdk-ruby
|
||||
- aws-sdk-java
|
||||
- mc
|
||||
- minio-go
|
||||
- minio-java
|
||||
- minio-js
|
||||
- minio-py
|
||||
- minio-dotnet
|
||||
- s3cmd
|
||||
- worm
|
||||
|
||||
## Running Mint
|
||||
|
||||
Mint is run by `docker run` command which requires Docker to be installed. For Docker installation follow the steps [here](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/).
|
||||
|
||||
To run Mint with Minio Play server as test target,
|
||||
|
||||
```sh
|
||||
$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \
|
||||
-e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -e ENABLE_HTTPS=1 minio/mint
|
||||
```
|
||||
|
||||
After the tests are run, output is stored in `/mint/log` directory inside the container. To get these logs, use `docker cp` command. For example
|
||||
```sh
|
||||
docker cp <container-id>:/mint/log /tmp/logs
|
||||
```
|
||||
|
||||
### Mint environment variables
|
||||
|
||||
Below environment variables are required to be passed to the docker container. Supported environment variables:
|
||||
|
||||
| Environment variable | Description | Example |
|
||||
|:--- |:--- |:--- |
|
||||
| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` |
|
||||
| `ACCESS_KEY` | Access key of access `SERVER_ENDPOINT` | `Q3AM3UQ867SPQQA43P2F` |
|
||||
| `SECRET_KEY` | Secret Key of access `SERVER_ENDPOINT` | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` |
|
||||
| `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` |
|
||||
| `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full` or `worm`. Defaults to `core` | `full` |
|
||||
| `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` |
|
||||
| `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` |
|
||||
|
||||
|
||||
### Test virtual style access against Minio server
|
||||
|
||||
To test Minio server virtual style access with Mint, follow these steps:
|
||||
|
||||
- Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`.
|
||||
- Start Minio server.
|
||||
- Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command
|
||||
```sh
|
||||
$ docker run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com" \
|
||||
-e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \
|
||||
-e "ENABLE_VIRTUAL_STYLE=1" minio/mint
|
||||
```
|
||||
|
||||
### Mint log format
|
||||
|
||||
All test logs are stored in `/mint/log/log.json` as multiple JSON document. Below is the JSON format for every entry in the log file.
|
||||
|
||||
| JSON field | Type | Description | Example |
|
||||
|:--- |:--- |:--- |:--- |
|
||||
| `name` | _string_ | Testing tool/SDK name | `"aws-sdk-php"` |
|
||||
| `function` | _string_ | Test function name | `"getBucketLocation ( array $params = [] )"` |
|
||||
| `args` | _object_ | (Optional) Key/Value map of arguments passed to test function | `{"Bucket":"aws-sdk-php-bucket-20341"}` |
|
||||
| `duration` | _int_ | Time taken in milliseconds to run the test | `384` |
|
||||
| `status` | _string_ | one of `PASS`, `FAIL` or `NA` | `"PASS"` |
|
||||
| `alert` | _string_ | (Optional) Alert message indicating test failure | `"I/O error on create file"` |
|
||||
| `message` | _string_ | (Optional) Any log message | `"validating checksum of downloaded object"` |
|
||||
| `error` | _string_ | Detailed error message including stack trace on status `FAIL` | `"Error executing \"CompleteMultipartUpload\" on ...` |
|
||||
|
||||
## For Developers
|
||||
|
||||
### Running Mint development code
|
||||
|
||||
After making changes to Mint source code a local docker image can be built/run by
|
||||
|
||||
```sh
|
||||
$ docker build -t minio/mint . -f Dockerfile.dev
|
||||
$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \
|
||||
-e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \
|
||||
-e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest
|
||||
```
|
||||
In case of Worm Mode, start your server with configuration `MINT_MODE` set to `worm`.
|
||||
Build/Run of local docker image for Worm mode, is to be tested against your server configuration, by
|
||||
|
||||
```
|
||||
### Adding tests with new tool/SDK
|
||||
|
||||
Below are the steps need to be followed
|
||||
|
||||
* Create new app directory under [build](https://github.com/minio/mint/tree/master/build) and [run/core](https://github.com/minio/mint/tree/master/run/core) directories.
|
||||
* Create `install.sh` which does installation of required tool/SDK under app directory.
|
||||
* Any build and install time dependencies should be added to [install-packages.list](https://github.com/minio/mint/blob/master/install-packages.list).
|
||||
* Build time dependencies should be added to [remove-packages.list](https://github.com/minio/mint/blob/master/remove-packages.list) for removal to have clean Mint docker image.
|
||||
* Add `run.sh` in app directory under `run/core` which execute actual tests.
|
||||
|
||||
#### Test data
|
||||
|
||||
Tests may use pre-created data set to perform various object operations on Minio server. Below data files are available under `/mint/data` directory.
|
||||
|
||||
| File name | Size |
|
||||
|:--- |:--- |
|
||||
| datafile-0-b | 0B |
|
||||
| datafile-1-b | 1B |
|
||||
| datafile-1-kB |1KiB |
|
||||
| datafile-10-kB |10KiB |
|
||||
| datafile-33-kB |33KiB |
|
||||
| datafile-100-kB |100KiB |
|
||||
| datafile-1-MB |1MiB |
|
||||
| datafile-1.03-MB |1.03MiB |
|
||||
| datafile-5-MB |5MiB |
|
||||
| datafile-6-MB |6MiB |
|
||||
| datafile-10-MB |10MiB |
|
||||
| datafile-11-MB |11MiB |
|
||||
| datafile-65-MB |65MiB |
|
||||
| datafile-129-MB |129MiB |
|
19
mint/build/aws-sdk-go/install.sh
Executable file
19
mint/build/aws-sdk-go/install.sh
Executable file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go"
|
||||
GO111MODULE=on go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go"
|
58
mint/build/aws-sdk-java/build.xml
Normal file
58
mint/build/aws-sdk-java/build.xml
Normal file
@ -0,0 +1,58 @@
|
||||
<project xmlns:ivy="antlib:org.apache.ivy.ant" name="aws-sdk-java-tests" default="run">
|
||||
<property name="ivy.install.version" value="2.1.0-rc2" />
|
||||
<condition property="ivy.home" value="${env.IVY_HOME}">
|
||||
<isset property="env.IVY_HOME" />
|
||||
</condition>
|
||||
<property name="ivy.home" value="${user.home}/.ant" />
|
||||
<property name="ivy.jar.dir" value="${ivy.home}/lib" />
|
||||
<property name="ivy.jar.file" value="${ivy.jar.dir}/ivy.jar" />
|
||||
|
||||
<target name="download-ivy" unless="offline">
|
||||
<mkdir dir="${ivy.jar.dir}"/>
|
||||
<get src="https://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/ivy-${ivy.install.version}.jar"
|
||||
dest="${ivy.jar.file}" usetimestamp="true"/>
|
||||
</target>
|
||||
|
||||
<target name="init-ivy" depends="download-ivy">
|
||||
<path id="ivy.lib.path">
|
||||
<fileset dir="${ivy.jar.dir}" includes="*.jar"/>
|
||||
|
||||
</path>
|
||||
<taskdef resource="org/apache/ivy/ant/antlib.xml"
|
||||
uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path"/>
|
||||
</target>
|
||||
<target name="resolve" description="--> retrieve dependencies with ivy">
|
||||
<ivy:retrieve />
|
||||
</target>
|
||||
|
||||
<target name="clean">
|
||||
<delete dir="build"/>
|
||||
</target>
|
||||
|
||||
<path id="aws-s3-sdk-deps">
|
||||
<fileset dir="lib">
|
||||
<include name="*.jar"/>
|
||||
</fileset>
|
||||
</path>
|
||||
|
||||
<target name="compile">
|
||||
<mkdir dir="build/classes"/>
|
||||
<javac srcdir="src" destdir="build/classes">
|
||||
<classpath refid="aws-s3-sdk-deps" />
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
<target name="jar">
|
||||
<mkdir dir="build/jar"/>
|
||||
<jar destfile="build/jar/FunctionalTests.jar" basedir="build/classes">
|
||||
<archives>
|
||||
<zips>
|
||||
<fileset dir="lib/" includes="*.jar"/>
|
||||
</zips>
|
||||
</archives>
|
||||
<manifest>
|
||||
<attribute name="Main-Class" value="io.minio.awssdk.tests.FunctionalTests"/>
|
||||
</manifest>
|
||||
</jar>
|
||||
</target>
|
||||
</project>
|
30
mint/build/aws-sdk-java/install.sh
Executable file
30
mint/build/aws-sdk-java/install.sh
Executable file
@ -0,0 +1,30 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-java"
|
||||
|
||||
cd "$(dirname "$(realpath "$0")")"
|
||||
|
||||
ant init-ivy && \
|
||||
ant resolve && \
|
||||
ant compile && \
|
||||
ant jar
|
||||
|
||||
cp build/jar/FunctionalTests.jar "$test_run_dir/"
|
||||
|
||||
rm -rf lib/ build/
|
||||
|
6
mint/build/aws-sdk-java/ivy.xml
Normal file
6
mint/build/aws-sdk-java/ivy.xml
Normal file
@ -0,0 +1,6 @@
|
||||
<ivy-module version="2.0">
|
||||
<info organisation="org.apache" module="aws-sdk-java-tests"/>
|
||||
<dependencies>
|
||||
<dependency org="com.amazonaws" name="aws-java-sdk-s3" rev="1.11.289"/>
|
||||
</dependencies>
|
||||
</ivy-module>
|
632
mint/build/aws-sdk-java/src/FunctionalTests.java
Normal file
632
mint/build/aws-sdk-java/src/FunctionalTests.java
Normal file
@ -0,0 +1,632 @@
|
||||
/*
|
||||
* Mint, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
*
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.minio.awssdk.tests;
|
||||
|
||||
import java.io.*;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.SecureRandom;
|
||||
|
||||
import java.security.*;
|
||||
import java.util.*;
|
||||
|
||||
import java.nio.file.*;
|
||||
import java.math.BigInteger;
|
||||
|
||||
import javax.crypto.KeyGenerator;
|
||||
import javax.crypto.SecretKey;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.AmazonServiceException;
|
||||
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import com.amazonaws.services.s3.model.CreateBucketRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectListing;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.amazonaws.services.s3.model.SSECustomerKey;
|
||||
|
||||
// Main Testing class
|
||||
public class FunctionalTests {
|
||||
|
||||
private static final String PASS = "PASS";
|
||||
private static final String FAILED = "FAIL";
|
||||
private static final String IGNORED = "NA";
|
||||
|
||||
private static String accessKey;
|
||||
private static String secretKey;
|
||||
private static String region;
|
||||
private static String endpoint;
|
||||
private static boolean enableHTTPS;
|
||||
|
||||
private static final Random random = new Random(new SecureRandom().nextLong());
|
||||
private static String bucketName = getRandomName();
|
||||
private static boolean mintEnv = false;
|
||||
|
||||
private static String file1Kb;
|
||||
private static String file1Mb;
|
||||
private static String file6Mb;
|
||||
|
||||
private static SSECustomerKey sseKey1;
|
||||
private static SSECustomerKey sseKey2;
|
||||
private static SSECustomerKey sseKey3;
|
||||
|
||||
private static AmazonS3 s3Client;
|
||||
private static S3TestUtils s3TestUtils;
|
||||
|
||||
public static String getRandomName() {
|
||||
return "aws-java-sdk-test-" + new BigInteger(32, random).toString(32);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints a success log entry in JSON format.
|
||||
*/
|
||||
public static void mintSuccessLog(String function, String args, long startTime) {
|
||||
if (mintEnv) {
|
||||
System.out.println(
|
||||
new MintLogger(function, args, System.currentTimeMillis() - startTime, PASS, null, null, null));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints a failure log entry in JSON format.
|
||||
*/
|
||||
public static void mintFailedLog(String function, String args, long startTime, String message, String error) {
|
||||
if (mintEnv) {
|
||||
System.out.println(new MintLogger(function, args, System.currentTimeMillis() - startTime, FAILED, null,
|
||||
message, error));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints a ignore log entry in JSON format.
|
||||
*/
|
||||
public static void mintIgnoredLog(String function, String args, long startTime) {
|
||||
if (mintEnv) {
|
||||
System.out.println(
|
||||
new MintLogger(function, args, System.currentTimeMillis() - startTime, IGNORED, null, null, null));
|
||||
}
|
||||
}
|
||||
|
||||
public static void initTests() throws IOException {
|
||||
// Create encryption key.
|
||||
byte[] rawKey1 = "32byteslongsecretkeymustgenerate".getBytes();
|
||||
SecretKey secretKey1 = new SecretKeySpec(rawKey1, 0, rawKey1.length, "AES");
|
||||
sseKey1 = new SSECustomerKey(secretKey1);
|
||||
|
||||
// Create new encryption key for target so it is saved using sse-c
|
||||
byte[] rawKey2 = "xxbytescopysecretkeymustprovided".getBytes();
|
||||
SecretKey secretKey2 = new SecretKeySpec(rawKey2, 0, rawKey2.length, "AES");
|
||||
sseKey2 = new SSECustomerKey(secretKey2);
|
||||
|
||||
// Create new encryption key for target so it is saved using sse-c
|
||||
byte[] rawKey3 = "32byteslongsecretkeymustgenerat1".getBytes();
|
||||
SecretKey secretKey3 = new SecretKeySpec(rawKey3, 0, rawKey3.length, "AES");
|
||||
sseKey3 = new SSECustomerKey(secretKey3);
|
||||
|
||||
// Create bucket
|
||||
s3Client.createBucket(new CreateBucketRequest(bucketName));
|
||||
}
|
||||
|
||||
public static void teardown() throws IOException {
|
||||
|
||||
// Remove all objects under the test bucket & the bucket itself
|
||||
// TODO: use multi delete API instead
|
||||
ObjectListing objectListing = s3Client.listObjects(bucketName);
|
||||
while (true) {
|
||||
for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) {
|
||||
S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
|
||||
s3Client.deleteObject(bucketName, summary.getKey());
|
||||
}
|
||||
// more objectListing to retrieve?
|
||||
if (objectListing.isTruncated()) {
|
||||
objectListing = s3Client.listNextBatchOfObjects(objectListing);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
;
|
||||
s3Client.deleteBucket(bucketName);
|
||||
}
|
||||
|
||||
// Test regular object upload using encryption
|
||||
public static void uploadObjectEncryption_test1() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println(
|
||||
"Test: uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
String file1KbMD5 = Utils.getFileMD5(file1Kb);
|
||||
String objectName = "testobject";
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
|
||||
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, file1KbMD5);
|
||||
mintSuccessLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb
|
||||
+ ", SSECustomerKey: " + sseKey1,
|
||||
startTime);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb
|
||||
+ ", SSECustomerKey: " + sseKey1,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
// Test downloading an object with a wrong encryption key
|
||||
public static void downloadObjectEncryption_test1() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
String file1KbMD5 = Utils.getFileMD5(file1Kb);
|
||||
String objectName = "testobject";
|
||||
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, "testobject", file1Kb, sseKey1);
|
||||
s3TestUtils.downloadObject(bucketName, objectName, sseKey2);
|
||||
Exception ex = new Exception("downloadObject did not throw an S3 Access denied exception");
|
||||
mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2,
|
||||
startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace()));
|
||||
throw ex;
|
||||
} catch (Exception e) {
|
||||
if (!e.getMessage().contains("Access Denied")) {
|
||||
Exception ex = new Exception(
|
||||
"downloadObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage());
|
||||
mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2,
|
||||
startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace()));
|
||||
throw ex;
|
||||
}
|
||||
mintSuccessLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2,
|
||||
startTime);
|
||||
}
|
||||
}
|
||||
|
||||
// Test copying object with a new different encryption key
|
||||
public static void copyObjectEncryption_test1() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
String file1KbMD5 = Utils.getFileMD5(file1Kb);
|
||||
String objectName = "testobject";
|
||||
String dstObjectName = "dir/newobject";
|
||||
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
|
||||
s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false);
|
||||
s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file1KbMD5);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
|
||||
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: "
|
||||
+ sseKey2 + ", replaceDirective: " + false,
|
||||
startTime);
|
||||
}
|
||||
|
||||
// Test copying object with wrong source encryption key
|
||||
public static void copyObjectEncryption_test2() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
String objectName = "testobject";
|
||||
String dstObjectName = "dir/newobject";
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try {
|
||||
s3TestUtils.copyObject(bucketName, objectName, sseKey3, bucketName, dstObjectName, sseKey2, false);
|
||||
Exception ex = new Exception("copyObject did not throw an S3 Access denied exception");
|
||||
mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3
|
||||
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
|
||||
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
|
||||
startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace()));
|
||||
throw ex;
|
||||
} catch (Exception e) {
|
||||
if (!e.getMessage().contains("Access Denied")) {
|
||||
Exception ex = new Exception(
|
||||
"copyObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage());
|
||||
mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3
|
||||
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
|
||||
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
|
||||
startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace()));
|
||||
throw ex;
|
||||
}
|
||||
mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3
|
||||
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
|
||||
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
|
||||
startTime);
|
||||
}
|
||||
}
|
||||
|
||||
// Test copying multipart object
|
||||
public static void copyObjectEncryption_test3() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
String file6MbMD5 = Utils.getFileMD5(file6Mb);
|
||||
String objectName = "testobject";
|
||||
String dstObjectName = "dir/newobject";
|
||||
|
||||
try {
|
||||
s3TestUtils.uploadMultipartObject(bucketName, objectName, file6Mb, sseKey1);
|
||||
s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false);
|
||||
s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file6MbMD5);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
|
||||
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
|
||||
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: "
|
||||
+ sseKey2 + ", replaceDirective: " + false,
|
||||
startTime);
|
||||
}
|
||||
|
||||
// Test downloading encrypted object with Get Range, 0 -> 1024
|
||||
public static void downloadGetRangeEncryption_test1() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
String objectName = "testobject";
|
||||
String range1MD5 = Utils.getFileMD5(file1Kb);
|
||||
int start = 0;
|
||||
int length = 1024;
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
|
||||
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
mintSuccessLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime);
|
||||
}
|
||||
|
||||
// Test downloading encrypted object with Get Range, 0 -> 1
|
||||
public static void downloadGetRangeEncryption_test2() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
String objectName = "testobject";
|
||||
int start = 0;
|
||||
int length = 1;
|
||||
String range1MD5 = Utils.getFileMD5(file1Kb, start, length);
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
|
||||
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
mintSuccessLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime);
|
||||
}
|
||||
|
||||
// Test downloading encrypted object with Get Range, 0 -> 1024-1
|
||||
public static void downloadGetRangeEncryption_test3() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
String objectName = "testobject";
|
||||
int start = 0;
|
||||
int length = 1023;
|
||||
String range1MD5 = Utils.getFileMD5(file1Kb, start, length);
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
|
||||
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
mintSuccessLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime);
|
||||
}
|
||||
|
||||
// Test downloading encrypted object with Get Range, 1 -> 1024-1
|
||||
public static void downloadGetRangeEncryption_test4() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
String objectName = "testobject";
|
||||
int start = 1;
|
||||
int length = 1023;
|
||||
String range1MD5 = Utils.getFileMD5(file1Kb, start, length);
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
|
||||
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
mintSuccessLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime);
|
||||
}
|
||||
|
||||
// Test downloading encrypted object with Get Range, 64*1024 -> 64*1024
|
||||
public static void downloadGetRangeEncryption_test5() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
String objectName = "testobject";
|
||||
int start = 64 * 1024;
|
||||
int length = 64 * 1024;
|
||||
String range1MD5 = Utils.getFileMD5(file1Mb, start, length);
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1);
|
||||
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
mintSuccessLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime);
|
||||
}
|
||||
|
||||
// Test downloading encrypted object with Get Range, 64*1024 ->
|
||||
// 1024*1024-64*1024
|
||||
public static void downloadGetRangeEncryption_test6() throws Exception {
|
||||
if (!mintEnv) {
|
||||
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
|
||||
}
|
||||
|
||||
if (!enableHTTPS) {
|
||||
return;
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
String objectName = "testobject";
|
||||
int start = 64 * 1024;
|
||||
int length = 1024 * 1024 - 64 * 1024;
|
||||
String range1MD5 = Utils.getFileMD5(file1Mb, start, length);
|
||||
try {
|
||||
s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1);
|
||||
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
|
||||
} catch (Exception e) {
|
||||
mintFailedLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
|
||||
throw e;
|
||||
}
|
||||
mintSuccessLog(
|
||||
"downloadObjectGetRange(String bucketName, String objectName, "
|
||||
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
|
||||
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
|
||||
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
|
||||
startTime);
|
||||
}
|
||||
|
||||
// Run tests
|
||||
public static void runTests() throws Exception {
|
||||
|
||||
uploadObjectEncryption_test1();
|
||||
|
||||
downloadObjectEncryption_test1();
|
||||
|
||||
copyObjectEncryption_test1();
|
||||
copyObjectEncryption_test2();
|
||||
copyObjectEncryption_test3();
|
||||
|
||||
downloadGetRangeEncryption_test1();
|
||||
downloadGetRangeEncryption_test2();
|
||||
downloadGetRangeEncryption_test3();
|
||||
downloadGetRangeEncryption_test4();
|
||||
downloadGetRangeEncryption_test5();
|
||||
downloadGetRangeEncryption_test6();
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception, IOException, NoSuchAlgorithmException {
|
||||
|
||||
endpoint = System.getenv("SERVER_ENDPOINT");
|
||||
accessKey = System.getenv("ACCESS_KEY");
|
||||
secretKey = System.getenv("SECRET_KEY");
|
||||
enableHTTPS = System.getenv("ENABLE_HTTPS").equals("1");
|
||||
|
||||
region = "us-east-1";
|
||||
|
||||
if (enableHTTPS) {
|
||||
endpoint = "https://" + endpoint;
|
||||
} else {
|
||||
endpoint = "http://" + endpoint;
|
||||
}
|
||||
|
||||
String dataDir = System.getenv("MINT_DATA_DIR");
|
||||
if (dataDir != null && !dataDir.equals("")) {
|
||||
mintEnv = true;
|
||||
file1Kb = Paths.get(dataDir, "datafile-1-kB").toString();
|
||||
file1Mb = Paths.get(dataDir, "datafile-1-MB").toString();
|
||||
file6Mb = Paths.get(dataDir, "datafile-6-MB").toString();
|
||||
}
|
||||
|
||||
String mintMode = null;
|
||||
if (mintEnv) {
|
||||
mintMode = System.getenv("MINT_MODE");
|
||||
}
|
||||
|
||||
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
|
||||
AmazonS3ClientBuilder.EndpointConfiguration endpointConfiguration = new AmazonS3ClientBuilder.EndpointConfiguration(
|
||||
endpoint, region);
|
||||
|
||||
AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
|
||||
clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials));
|
||||
clientBuilder.setEndpointConfiguration(endpointConfiguration);
|
||||
clientBuilder.setPathStyleAccessEnabled(true);
|
||||
|
||||
s3Client = clientBuilder.build();
|
||||
s3TestUtils = new S3TestUtils(s3Client);
|
||||
|
||||
try {
|
||||
initTests();
|
||||
FunctionalTests.runTests();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
System.exit(-1);
|
||||
} finally {
|
||||
teardown();
|
||||
}
|
||||
}
|
||||
}
|
60
mint/build/aws-sdk-java/src/LimitedInputStream.java
Normal file
60
mint/build/aws-sdk-java/src/LimitedInputStream.java
Normal file
@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Mint, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
*
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.minio.awssdk.tests;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
// LimitedInputStream wraps a regular InputStream, calling
|
||||
// read() will skip some bytes as configured and will also
|
||||
// return only data with configured length
|
||||
|
||||
class LimitedInputStream extends InputStream {
|
||||
|
||||
private int skip;
|
||||
private int length;
|
||||
private InputStream is;
|
||||
|
||||
LimitedInputStream(InputStream is, int skip, int length) {
|
||||
this.is = is;
|
||||
this.skip = skip;
|
||||
this.length = length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
int r;
|
||||
while (skip > 0) {
|
||||
r = is.read();
|
||||
if (r < 0) {
|
||||
throw new IOException("stream ended before being able to skip all bytes");
|
||||
}
|
||||
skip--;
|
||||
}
|
||||
if (length == 0) {
|
||||
return -1;
|
||||
}
|
||||
r = is.read();
|
||||
if (r < 0) {
|
||||
throw new IOException("stream ended before being able to read all bytes");
|
||||
}
|
||||
length--;
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
|
151
mint/build/aws-sdk-java/src/MintLogger.java
Executable file
151
mint/build/aws-sdk-java/src/MintLogger.java
Executable file
@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Minio Java SDK for Amazon S3 Compatible Cloud Storage,
|
||||
* (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.minio.awssdk.tests;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAutoDetect;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude.Include;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
|
||||
public class MintLogger {
|
||||
|
||||
@JsonProperty("name")
|
||||
private String name;
|
||||
|
||||
@JsonProperty("function")
|
||||
private String function;
|
||||
|
||||
@JsonProperty("args")
|
||||
private String args;
|
||||
|
||||
@JsonProperty("duration")
|
||||
private long duration;
|
||||
|
||||
@JsonProperty("status")
|
||||
private String status;
|
||||
|
||||
@JsonProperty("alert")
|
||||
private String alert;
|
||||
|
||||
@JsonProperty("message")
|
||||
private String message;
|
||||
|
||||
@JsonProperty("error")
|
||||
private String error;
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
**/
|
||||
public MintLogger(String function,
|
||||
String args,
|
||||
long duration,
|
||||
String status,
|
||||
String alert,
|
||||
String message,
|
||||
String error) {
|
||||
this.name = "aws-sdk-java";
|
||||
this.function = function;
|
||||
this.duration = duration;
|
||||
this.args = args;
|
||||
this.status = status;
|
||||
this.alert = alert;
|
||||
this.message = message;
|
||||
this.error = error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return JSON Log Entry.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public String toString() {
|
||||
|
||||
try {
|
||||
return new ObjectMapper().setSerializationInclusion(Include.NON_NULL).writeValueAsString(this);
|
||||
} catch (JsonProcessingException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Return Alert.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public String alert() {
|
||||
return alert;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return Error.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public String error() {
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return Message.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public String message() {
|
||||
return message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return args.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public String args() {
|
||||
return args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return status.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public String status() {
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return name.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public String name() {
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return function.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public String function() {
|
||||
return function;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return duration.
|
||||
**/
|
||||
@JsonIgnore
|
||||
public long duration() {
|
||||
return duration;
|
||||
}
|
||||
}
|
187
mint/build/aws-sdk-java/src/S3TestUtils.java
Normal file
187
mint/build/aws-sdk-java/src/S3TestUtils.java
Normal file
@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Mint, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
*
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.minio.awssdk.tests;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
import java.nio.channels.Channels;
|
||||
|
||||
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||
import com.amazonaws.services.s3.model.CopyObjectRequest;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.model.S3ObjectInputStream;
|
||||
import com.amazonaws.services.s3.model.SSECustomerKey;
|
||||
|
||||
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
|
||||
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
|
||||
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
|
||||
import com.amazonaws.services.s3.model.PartETag;
|
||||
import com.amazonaws.services.s3.model.UploadPartRequest;
|
||||
|
||||
import com.amazonaws.services.s3.model.MetadataDirective;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
|
||||
class S3TestUtils {
|
||||
|
||||
private AmazonS3 s3Client;
|
||||
|
||||
S3TestUtils(AmazonS3 s3Client) {
|
||||
this.s3Client = s3Client;
|
||||
}
|
||||
|
||||
void uploadMultipartObject(String bucketName, String keyName,
|
||||
String filePath, SSECustomerKey sseKey) throws IOException {
|
||||
|
||||
File file = new File(filePath);
|
||||
|
||||
List<PartETag> partETags = new ArrayList<PartETag>();
|
||||
|
||||
// Step 1: Initialize.
|
||||
InitiateMultipartUploadRequest initRequest = new
|
||||
InitiateMultipartUploadRequest(bucketName, keyName);
|
||||
|
||||
if (sseKey != null) {
|
||||
initRequest.setSSECustomerKey(sseKey);
|
||||
}
|
||||
|
||||
InitiateMultipartUploadResult initResponse =
|
||||
s3Client.initiateMultipartUpload(initRequest);
|
||||
|
||||
long contentLength = file.length();
|
||||
long partSize = 5242880; // Set part size to 5 MB.
|
||||
|
||||
// Step 2: Upload parts.
|
||||
long filePosition = 0;
|
||||
for (int i = 1; filePosition < contentLength; i++) {
|
||||
// Last part can be less than 5 MB. Adjust part size.
|
||||
partSize = Math.min(partSize, (contentLength - filePosition));
|
||||
|
||||
// Create request to upload a part.
|
||||
UploadPartRequest uploadRequest = new UploadPartRequest()
|
||||
.withBucketName(bucketName).withKey(keyName)
|
||||
.withUploadId(initResponse.getUploadId()).withPartNumber(i)
|
||||
.withFileOffset(filePosition)
|
||||
.withFile(file)
|
||||
.withPartSize(partSize);
|
||||
|
||||
if (sseKey != null) {
|
||||
uploadRequest.withSSECustomerKey(sseKey);
|
||||
}
|
||||
|
||||
// Upload part and add response to our list.
|
||||
partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
|
||||
|
||||
filePosition += partSize;
|
||||
}
|
||||
|
||||
// Step 3: Complete.
|
||||
CompleteMultipartUploadRequest compRequest = new
|
||||
CompleteMultipartUploadRequest(
|
||||
bucketName,
|
||||
keyName,
|
||||
initResponse.getUploadId(),
|
||||
partETags);
|
||||
|
||||
s3Client.completeMultipartUpload(compRequest);
|
||||
}
|
||||
|
||||
void uploadObject(String bucketName, String keyName,
|
||||
String filePath, SSECustomerKey sseKey) throws IOException {
|
||||
|
||||
File f = new File(filePath);
|
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, f);
|
||||
if (sseKey != null) {
|
||||
putObjectRequest.withSSECustomerKey(sseKey);
|
||||
}
|
||||
s3Client.putObject(putObjectRequest);
|
||||
}
|
||||
|
||||
void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey)
|
||||
throws Exception, IOException {
|
||||
downloadObject(bucketName, keyName, sseKey, "", -1, -1);
|
||||
}
|
||||
|
||||
void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey,
|
||||
String expectedMD5)
|
||||
throws Exception, IOException {
|
||||
downloadObject(bucketName, keyName, sseKey, expectedMD5, -1, -1);
|
||||
}
|
||||
|
||||
void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey,
|
||||
String expectedMD5, int start, int length) throws Exception, IOException {
|
||||
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, keyName)
|
||||
.withSSECustomerKey(sseKey);
|
||||
|
||||
if (start >= 0 && length >= 0) {
|
||||
getObjectRequest.setRange(start, start+length-1);
|
||||
}
|
||||
|
||||
S3Object s3Object = s3Client.getObject(getObjectRequest);
|
||||
|
||||
int size = 0;
|
||||
int c;
|
||||
|
||||
S3ObjectInputStream input = s3Object.getObjectContent();
|
||||
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
String data = "";
|
||||
while ((c = input.read()) != -1) {
|
||||
output.write((byte) c);
|
||||
size++;
|
||||
}
|
||||
|
||||
if (length >= 0 && size != length) {
|
||||
throw new Exception("downloaded object has unexpected size, expected: " + length + ", received: " + size);
|
||||
}
|
||||
|
||||
String calculatedMD5 = Utils.getBufferMD5(output.toByteArray());
|
||||
|
||||
if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) {
|
||||
throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: " + calculatedMD5);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void copyObject(String bucketName, String keyName, SSECustomerKey sseKey,
|
||||
String targetBucketName, String targetKeyName, SSECustomerKey newSseKey,
|
||||
boolean replace) {
|
||||
CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, keyName, targetBucketName, targetKeyName);
|
||||
if (sseKey != null) {
|
||||
copyRequest.withSourceSSECustomerKey(sseKey);
|
||||
}
|
||||
if (newSseKey != null) {
|
||||
copyRequest.withDestinationSSECustomerKey(newSseKey);
|
||||
}
|
||||
if (replace) {
|
||||
copyRequest.withMetadataDirective(MetadataDirective.COPY);
|
||||
}
|
||||
s3Client.copyObject(copyRequest);
|
||||
}
|
||||
|
||||
long retrieveObjectMetadata(String bucketName, String keyName, SSECustomerKey sseKey) {
|
||||
GetObjectMetadataRequest getMetadataRequest = new GetObjectMetadataRequest(bucketName, keyName)
|
||||
.withSSECustomerKey(sseKey);
|
||||
ObjectMetadata objectMetadata = s3Client.getObjectMetadata(getMetadataRequest);
|
||||
return objectMetadata.getContentLength();
|
||||
}
|
||||
|
||||
}
|
76
mint/build/aws-sdk-java/src/Utils.java
Normal file
76
mint/build/aws-sdk-java/src/Utils.java
Normal file
@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Mint, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
*
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
package io.minio.awssdk.tests;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.channels.*;
|
||||
import java.security.*;
|
||||
|
||||
class Utils {
|
||||
|
||||
public static byte[] createChecksum(InputStream is, int skip, int length) throws Exception {
|
||||
int numRead;
|
||||
byte[] buffer = new byte[1024];
|
||||
|
||||
MessageDigest complete = MessageDigest.getInstance("MD5");
|
||||
|
||||
if (skip > -1 && length > -1) {
|
||||
is = new LimitedInputStream(is, skip, length);
|
||||
}
|
||||
|
||||
do {
|
||||
numRead = is.read(buffer);
|
||||
if (numRead > 0) {
|
||||
complete.update(buffer, 0, numRead);
|
||||
}
|
||||
} while (numRead != -1);
|
||||
|
||||
return complete.digest();
|
||||
}
|
||||
|
||||
public static String getInputStreamMD5(InputStream is) throws Exception {
|
||||
return getInputStreamMD5(is, -1, -1);
|
||||
}
|
||||
|
||||
public static String getInputStreamMD5(InputStream is, int start, int length) throws Exception {
|
||||
byte[] b = createChecksum(is, start, length);
|
||||
String result = "";
|
||||
|
||||
for (int i=0; i < b.length; i++) {
|
||||
result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 );
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static String getFileMD5(String filePath) throws Exception {
|
||||
return getFileMD5(filePath, -1, -1);
|
||||
}
|
||||
|
||||
public static String getFileMD5(String filePath, int start, int length) throws Exception {
|
||||
File f = new File(filePath);
|
||||
InputStream is = new FileInputStream(f);
|
||||
return getInputStreamMD5(is, start, length);
|
||||
}
|
||||
|
||||
public static String getBufferMD5(byte[] data) throws Exception {
|
||||
ByteArrayInputStream bis = new ByteArrayInputStream(data);
|
||||
return getInputStreamMD5(bis);
|
||||
}
|
||||
}
|
||||
|
20
mint/build/aws-sdk-php/install.sh
Executable file
20
mint/build/aws-sdk-php/install.sh
Executable file
@ -0,0 +1,20 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-php"
|
||||
$WGET --output-document=- https://getcomposer.org/installer | php -- --install-dir="$test_run_dir"
|
||||
php "$test_run_dir/composer.phar" --working-dir="$test_run_dir" install
|
18
mint/build/aws-sdk-ruby/install.sh
Executable file
18
mint/build/aws-sdk-ruby/install.sh
Executable file
@ -0,0 +1,18 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
gem install --no-rdoc --no-ri aws-sdk multipart_body
|
20
mint/build/awscli/install.sh
Executable file
20
mint/build/awscli/install.sh
Executable file
@ -0,0 +1,20 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
AWS_CLI_VERSION="1.11.112"
|
||||
|
||||
python -m pip install awscli==$AWS_CLI_VERSION
|
19
mint/build/healthcheck/install.sh
Executable file
19
mint/build/healthcheck/install.sh
Executable file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2019 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/healthcheck"
|
||||
GO111MODULE=on go build -o "$test_run_dir/healthcheck" "$test_run_dir/healthcheck.go"
|
31
mint/build/mc/install.sh
Executable file
31
mint/build/mc/install.sh
Executable file
@ -0,0 +1,31 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
MC_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/mc/releases/latest | sed "s/https:\/\/github.com\/minio\/mc\/releases\/tag\///")
|
||||
if [ -z "$MC_VERSION" ]; then
|
||||
echo "unable to get mc version from github"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/mc"
|
||||
$WGET --output-document="${test_run_dir}/mc" "https://dl.minio.io/client/mc/release/linux-amd64/mc.${MC_VERSION}"
|
||||
chmod a+x "${test_run_dir}/mc"
|
||||
|
||||
git clone --quiet https://github.com/minio/mc.git "$test_run_dir/mc.git"
|
||||
(cd "$test_run_dir/mc.git"; git checkout --quiet "tags/${MC_VERSION}")
|
||||
cp -a "${test_run_dir}/mc.git/functional-tests.sh" "$test_run_dir/"
|
||||
rm -fr "$test_run_dir/mc.git"
|
42
mint/build/minio-dotnet/install.sh
Executable file
42
mint/build/minio-dotnet/install.sh
Executable file
@ -0,0 +1,42 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
MINIO_DOTNET_SDK_PATH="$MINT_RUN_CORE_DIR/minio-dotnet"
|
||||
|
||||
MINIO_DOTNET_SDK_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-dotnet/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-dotnet\/releases\/tag\///")
|
||||
if [ -z "$MINIO_DOTNET_SDK_VERSION" ]; then
|
||||
echo "unable to get minio-dotnet version from github"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
out_dir="$MINIO_DOTNET_SDK_PATH/out"
|
||||
if [ -z "$out_dir" ]; then
|
||||
mkdir "$out_dir"
|
||||
fi
|
||||
|
||||
curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/FunctionalTest.cs > "${MINIO_DOTNET_SDK_PATH}/FunctionalTest.cs"
|
||||
curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/MintLogger.cs > "${MINIO_DOTNET_SDK_PATH}/MintLogger.cs"
|
||||
curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/JsonNetLogger.cs > "${MINIO_DOTNET_SDK_PATH}/JsonNetLogger.cs"
|
||||
curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/Program.cs > "${MINIO_DOTNET_SDK_PATH}/Program.cs"
|
||||
curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/RandomStreamGenerator.cs > "${MINIO_DOTNET_SDK_PATH}/RandomStreamGenerator.cs"
|
||||
curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/Minio.Functional.Tests.csproj > "${MINIO_DOTNET_SDK_PATH}/Minio.Functional.Tests.csproj"
|
||||
|
||||
cd "$MINIO_DOTNET_SDK_PATH"
|
||||
dotnet restore /p:Configuration=Mint
|
||||
dotnet publish --runtime ubuntu.16.04-x64 --output out /p:Configuration=Mint
|
27
mint/build/minio-go/install.sh
Executable file
27
mint/build/minio-go/install.sh
Executable file
@ -0,0 +1,27 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
MINIO_GO_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-go/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-go\/releases\/tag\///")
|
||||
if [ -z "$MINIO_GO_VERSION" ]; then
|
||||
echo "unable to get minio-go version from github"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/minio-go"
|
||||
(git clone https://github.com/minio/minio-go && cd minio-go && git checkout --quiet "tags/$MINIO_GO_VERSION")
|
||||
GO111MODULE=on CGO_ENABLED=0 go build -o "$test_run_dir/minio-go" "minio-go/functional_tests.go"
|
||||
rm -rf minio-go
|
30
mint/build/minio-java/install.sh
Executable file
30
mint/build/minio-java/install.sh
Executable file
@ -0,0 +1,30 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
MINIO_JAVA_VERSION=$(curl --retry 10 -s "http://repo1.maven.org/maven2/io/minio/minio/maven-metadata.xml" | sed -n "/<latest>/{s/<.[^>]*>//g;p;q}" | sed "s/ *//g")
|
||||
if [ -z "$MINIO_JAVA_VERSION" ]; then
|
||||
echo "unable to get latest minio-java version from maven"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/minio-java"
|
||||
git clone --quiet https://github.com/minio/minio-java.git "$test_run_dir/minio-java.git"
|
||||
(cd "$test_run_dir/minio-java.git"; git checkout --quiet "tags/${MINIO_JAVA_VERSION}")
|
||||
$WGET --output-document="$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "http://repo1.maven.org/maven2/io/minio/minio/${MINIO_JAVA_VERSION}/minio-${MINIO_JAVA_VERSION}-all.jar"
|
||||
javac -cp "$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "${test_run_dir}/minio-java.git/functional"/*.java
|
||||
cp -a "${test_run_dir}/minio-java.git/functional"/*.class "$test_run_dir/"
|
||||
rm -fr "$test_run_dir/minio-java.git"
|
28
mint/build/minio-js/install.sh
Executable file
28
mint/build/minio-js/install.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
MINIO_JS_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-js/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-js\/releases\/tag\///")
|
||||
if [ -z "$MINIO_JS_VERSION" ]; then
|
||||
echo "unable to get minio-js version from github"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/minio-js"
|
||||
mkdir "${test_run_dir}/test"
|
||||
$WGET --output-document="${test_run_dir}/test/functional-tests.js" "https://raw.githubusercontent.com/minio/minio-js/${MINIO_JS_VERSION}/src/test/functional/functional-tests.js"
|
||||
npm --prefix "$test_run_dir" install --save "minio@$MINIO_JS_VERSION"
|
||||
npm --prefix "$test_run_dir" install
|
29
mint/build/minio-py/install.sh
Executable file
29
mint/build/minio-py/install.sh
Executable file
@ -0,0 +1,29 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
MINIO_PY_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-py/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-py\/releases\/tag\///")
|
||||
if [ -z "$MINIO_PY_VERSION" ]; then
|
||||
echo "unable to get minio-py version from github"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/minio-py"
|
||||
# FIX https://github.com/pypa/pip/issues/5221
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install --user faker
|
||||
python -m pip install minio=="$MINIO_PY_VERSION"
|
||||
$WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/${MINIO_PY_VERSION}/tests/functional/tests.py"
|
19
mint/build/s3cmd/install.sh
Executable file
19
mint/build/s3cmd/install.sh
Executable file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Always install the latest.
|
||||
python -m pip install s3cmd
|
19
mint/build/security/install.sh
Executable file
19
mint/build/security/install.sh
Executable file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2018 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/security"
|
||||
GO111MODULE=on go build -o "$test_run_dir/tls-tests" "$test_run_dir/tls-tests.go"
|
19
mint/build/worm/install.sh
Executable file
19
mint/build/worm/install.sh
Executable file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
test_run_dir="$MINT_RUN_CORE_DIR/worm"
|
||||
GO111MODULE=on CGO_ENABLED=0 go build -o "$test_run_dir/worm" "$test_run_dir/quick-worm-tests.go"
|
44
mint/create-data-files.sh
Executable file
44
mint/create-data-files.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
MINT_DATA_DIR="$MINT_ROOT_DIR/data"
|
||||
|
||||
declare -A data_file_map
|
||||
data_file_map["datafile-0-b"]="0"
|
||||
data_file_map["datafile-1-b"]="1"
|
||||
data_file_map["datafile-1-kB"]="1K"
|
||||
data_file_map["datafile-10-kB"]="10K"
|
||||
data_file_map["datafile-33-kB"]="33K"
|
||||
data_file_map["datafile-100-kB"]="100K"
|
||||
data_file_map["datafile-1.03-MB"]="1056K"
|
||||
data_file_map["datafile-1-MB"]="1M"
|
||||
data_file_map["datafile-5-MB"]="5M"
|
||||
data_file_map["datafile-5243880-b"]="5243880"
|
||||
data_file_map["datafile-6-MB"]="6M"
|
||||
data_file_map["datafile-10-MB"]="10M"
|
||||
data_file_map["datafile-11-MB"]="11M"
|
||||
data_file_map["datafile-65-MB"]="65M"
|
||||
data_file_map["datafile-129-MB"]="129M"
|
||||
|
||||
mkdir -p "$MINT_DATA_DIR"
|
||||
for filename in "${!data_file_map[@]}"; do
|
||||
echo "creating $MINT_DATA_DIR/$filename"
|
||||
if ! shred -n 1 -s "${data_file_map[$filename]}" - 1>"$MINT_DATA_DIR/$filename" 2>/dev/null; then
|
||||
echo "unable to create data file $MINT_DATA_DIR/$filename"
|
||||
exit 1
|
||||
fi
|
||||
done
|
24
mint/entrypoint.sh
Executable file
24
mint/entrypoint.sh
Executable file
@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
./mint.sh "$@" &
|
||||
|
||||
# Get the pid to be used for kill command if required
|
||||
main_pid="$!"
|
||||
trap 'echo -e "\nAborting Mint..."; kill $main_pid' SIGINT SIGTERM
|
||||
# use -n here to catch mint.sh exit code, notify to ci
|
||||
wait -n
|
17
mint/install-packages.list
Normal file
17
mint/install-packages.list
Normal file
@ -0,0 +1,17 @@
|
||||
git
|
||||
python3-pip
|
||||
nodejs
|
||||
ruby
|
||||
ruby-dev
|
||||
ruby-bundler
|
||||
php
|
||||
php7.0-curl
|
||||
php-xml
|
||||
default-jre
|
||||
default-jdk
|
||||
ant
|
||||
dirmngr
|
||||
dotnet-sdk-2.1
|
||||
ca-certificates-mono
|
||||
nuget
|
||||
libunwind8
|
206
mint/mint.sh
Executable file
206
mint/mint.sh
Executable file
@ -0,0 +1,206 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017, 2018 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
CONTAINER_ID=$(awk -F / '{ print substr($NF, 1, 12) }' /proc/1/cpuset)
|
||||
MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data}
|
||||
MINT_MODE=${MINT_MODE:-core}
|
||||
SERVER_REGION=${SERVER_REGION:-us-east-1}
|
||||
ENABLE_HTTPS=${ENABLE_HTTPS:-0}
|
||||
ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0}
|
||||
GO111MODULE=on
|
||||
|
||||
if [ -z "$SERVER_ENDPOINT" ]; then
|
||||
SERVER_ENDPOINT="play.minio.io:9000"
|
||||
ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
ENABLE_HTTPS=1
|
||||
fi
|
||||
|
||||
if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then
|
||||
SERVER_IP="${SERVER_ENDPOINT%%:*}"
|
||||
SERVER_PORT="${SERVER_ENDPOINT/*:/}"
|
||||
# Check if SERVER_IP is actually IPv4 address
|
||||
octets=("${SERVER_IP//./ }")
|
||||
if [ "${#octets[@]}" -ne 4 ]; then
|
||||
echo "$SERVER_IP must be an IP address"
|
||||
exit 1
|
||||
fi
|
||||
for octet in "${octets[@]}"; do
|
||||
if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then
|
||||
echo "$SERVER_IP must be an IP address"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
ROOT_DIR="$PWD"
|
||||
TESTS_DIR="$ROOT_DIR/run/core"
|
||||
|
||||
BASE_LOG_DIR="$ROOT_DIR/log"
|
||||
LOG_FILE="log.json"
|
||||
ERROR_FILE="error.log"
|
||||
mkdir -p "$BASE_LOG_DIR"
|
||||
|
||||
function humanize_time()
|
||||
{
|
||||
time="$1"
|
||||
days=$(( time / 60 / 60 / 24 ))
|
||||
hours=$(( time / 60 / 60 % 24 ))
|
||||
minutes=$(( time / 60 % 60 ))
|
||||
seconds=$(( time % 60 ))
|
||||
|
||||
(( days > 0 )) && echo -n "$days days "
|
||||
(( hours > 0 )) && echo -n "$hours hours "
|
||||
(( minutes > 0 )) && echo -n "$minutes minutes "
|
||||
(( days > 0 || hours > 0 || minutes > 0 )) && echo -n "and "
|
||||
echo "$seconds seconds"
|
||||
}
|
||||
|
||||
function run_test()
|
||||
{
|
||||
if [ ! -d "$1" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
start=$(date +%s)
|
||||
|
||||
mkdir -p "$BASE_LOG_DIR/$sdk_name"
|
||||
|
||||
(cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE")
|
||||
rv=$?
|
||||
end=$(date +%s)
|
||||
duration=$(humanize_time $(( end - start )))
|
||||
|
||||
if [ "$rv" -eq 0 ]; then
|
||||
echo "done in $duration"
|
||||
else
|
||||
echo "FAILED in $duration"
|
||||
entry=$(tail -n 1 "$BASE_LOG_DIR/$LOG_FILE")
|
||||
status=$(jq -e -r .status <<<"$entry")
|
||||
jq_rv=$?
|
||||
if [ "$jq_rv" -ne 0 ]; then
|
||||
echo "$entry"
|
||||
fi
|
||||
## Show error.log when status is empty or not "FAIL".
|
||||
## This may happen when test run failed without providing logs.
|
||||
if [ "$jq_rv" -ne 0 ] || [ -z "$status" ] || ([ "$status" != "FAIL" ] && [ "$status" != "fail" ]); then
|
||||
cat "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE"
|
||||
else
|
||||
jq . <<<"$entry"
|
||||
fi
|
||||
fi
|
||||
return $rv
|
||||
}
|
||||
|
||||
function trust_s3_endpoint_tls_cert()
|
||||
{
|
||||
# Download the public certificate from the server
|
||||
openssl s_client -showcerts -connect "$SERVER_ENDPOINT" </dev/null 2>/dev/null | \
|
||||
openssl x509 -outform PEM -out /usr/local/share/ca-certificates/s3_server_cert.crt || \
|
||||
exit -1
|
||||
|
||||
# Load the certificate in the system
|
||||
update-ca-certificates --fresh >/dev/null
|
||||
|
||||
# Ask different SDKs/tools to load system certificates
|
||||
export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
|
||||
export NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
|
||||
export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
|
||||
}
|
||||
|
||||
|
||||
function main()
|
||||
{
|
||||
export MINT_DATA_DIR
|
||||
export MINT_MODE
|
||||
export SERVER_ENDPOINT
|
||||
export SERVER_IP
|
||||
export SERVER_PORT
|
||||
|
||||
export ACCESS_KEY
|
||||
export SECRET_KEY
|
||||
export ENABLE_HTTPS
|
||||
export SERVER_REGION
|
||||
export ENABLE_VIRTUAL_STYLE
|
||||
export GO111MODULE
|
||||
|
||||
echo "Running with"
|
||||
echo "SERVER_ENDPOINT: $SERVER_ENDPOINT"
|
||||
echo "ACCESS_KEY: $ACCESS_KEY"
|
||||
echo "SECRET_KEY: ***REDACTED***"
|
||||
echo "ENABLE_HTTPS: $ENABLE_HTTPS"
|
||||
echo "SERVER_REGION: $SERVER_REGION"
|
||||
echo "MINT_DATA_DIR: $MINT_DATA_DIR"
|
||||
echo "MINT_MODE: $MINT_MODE"
|
||||
echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE"
|
||||
echo
|
||||
echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'"
|
||||
echo
|
||||
|
||||
[ "$ENABLE_HTTPS" == "1" ] && trust_s3_endpoint_tls_cert
|
||||
|
||||
declare -a run_list
|
||||
if [ "$MINT_MODE" == "worm" ]; then
|
||||
if [ "$#" -gt 1 ]; then
|
||||
echo "No argument is accepted for worm mode"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_list=( "$TESTS_DIR/worm" )
|
||||
else
|
||||
sdks=( "$@" )
|
||||
|
||||
## populate all sdks except worm when no argument is given.
|
||||
if [ "$#" -eq 0 ]; then
|
||||
sdks=( $(ls -I worm "$TESTS_DIR") )
|
||||
fi
|
||||
|
||||
for sdk in "${sdks[@]}"; do
|
||||
if [ "$sdk" == "worm" ]; then
|
||||
echo "worm test cannot be run without worm mode"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_list=( "${run_list[@]}" "$TESTS_DIR/$sdk" )
|
||||
done
|
||||
fi
|
||||
|
||||
count="${#run_list[@]}"
|
||||
i=0
|
||||
for sdk_dir in "${run_list[@]}"; do
|
||||
sdk_name=$(basename "$sdk_dir")
|
||||
(( i++ ))
|
||||
if [ ! -d "$sdk_dir" ]; then
|
||||
echo "Test $sdk_name not found. Exiting Mint."
|
||||
exit 1
|
||||
fi
|
||||
echo -n "($i/$count) Running $sdk_name tests ... "
|
||||
if ! run_test "$sdk_dir"; then
|
||||
(( i-- ))
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
## Report when all tests in run_list are run
|
||||
if [ $i -eq "$count" ]; then
|
||||
echo -e "\nAll tests ran successfully"
|
||||
else
|
||||
echo -e "\nExecuted $i out of $count tests successfully."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
main "$@"
|
26
mint/postinstall.sh
Executable file
26
mint/postinstall.sh
Executable file
@ -0,0 +1,26 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# remove all packages listed in remove-packages.list
|
||||
xargs --arg-file=remove-packages.list apt --quiet --yes purge
|
||||
apt --quiet --yes autoremove
|
||||
|
||||
# remove unwanted files
|
||||
rm -fr "$GOROOT" "$GOPATH/src" /var/lib/apt/lists/*
|
||||
|
||||
# flush to disk
|
||||
sync
|
49
mint/preinstall.sh
Executable file
49
mint/preinstall.sh
Executable file
@ -0,0 +1,49 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
export APT="apt --quiet --yes"
|
||||
export WGET="wget --quiet --no-check-certificate"
|
||||
|
||||
# install nodejs source list
|
||||
if ! $WGET --output-document=- https://deb.nodesource.com/setup_6.x | bash -; then
|
||||
echo "unable to set nodejs repository"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
$APT install apt-transport-https
|
||||
|
||||
wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
rm -f packages-microsoft-prod.deb
|
||||
$APT install apt-transport-https
|
||||
$APT update
|
||||
|
||||
# download and install golang
|
||||
GO_VERSION="1.12.5"
|
||||
GO_INSTALL_PATH="/usr/local"
|
||||
download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz"
|
||||
if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then
|
||||
echo "unable to install go$GO_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
xargs --arg-file=install-packages.list apt --quiet --yes install
|
||||
|
||||
# set python 3.5 as default
|
||||
update-alternatives --install /usr/bin/python python /usr/bin/python3.5 1
|
||||
|
||||
sync
|
32
mint/release.sh
Executable file
32
mint/release.sh
Executable file
@ -0,0 +1,32 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
|
||||
export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
|
||||
export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
|
||||
export WGET="wget --quiet --no-check-certificate"
|
||||
|
||||
./create-data-files.sh
|
||||
./preinstall.sh
|
||||
|
||||
# install mint app packages
|
||||
for pkg in "$MINT_ROOT_DIR/build"/*/install.sh; do
|
||||
echo "Running $pkg"
|
||||
$pkg
|
||||
done
|
||||
|
||||
./postinstall.sh
|
9
mint/remove-packages.list
Normal file
9
mint/remove-packages.list
Normal file
@ -0,0 +1,9 @@
|
||||
wget
|
||||
git
|
||||
python3-pip
|
||||
ruby-dev
|
||||
ruby-bundler
|
||||
default-jdk
|
||||
ant
|
||||
dotnet
|
||||
nuget
|
328
mint/run/core/aws-sdk-go/quick-tests.go
Normal file
328
mint/run/core/aws-sdk-go/quick-tests.go
Normal file
@ -0,0 +1,328 @@
|
||||
/*
|
||||
*
|
||||
* Mint, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
|
||||
const (
|
||||
letterIdxBits = 6 // 6 bits to represent a letter index
|
||||
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
|
||||
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
|
||||
)
|
||||
const (
|
||||
PASS = "PASS" // Indicate that a test passed
|
||||
FAIL = "FAIL" // Indicate that a test failed
|
||||
NA = "NA" // Indicate that a test is not applicable
|
||||
)
|
||||
|
||||
type ErrorResponse struct {
|
||||
XMLName xml.Name `xml:"Error" json:"-"`
|
||||
Code string
|
||||
Message string
|
||||
BucketName string
|
||||
Key string
|
||||
RequestID string `xml:"RequestId"`
|
||||
HostID string `xml:"HostId"`
|
||||
|
||||
// Region where the bucket is located. This header is returned
|
||||
// only in HEAD bucket and ListObjects response.
|
||||
Region string
|
||||
|
||||
// Headers of the returned S3 XML error
|
||||
Headers http.Header `xml:"-" json:"-"`
|
||||
}
|
||||
|
||||
type mintJSONFormatter struct {
|
||||
}
|
||||
|
||||
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
|
||||
data := make(log.Fields, len(entry.Data))
|
||||
for k, v := range entry.Data {
|
||||
switch v := v.(type) {
|
||||
case error:
|
||||
// Otherwise errors are ignored by `encoding/json`
|
||||
// https://github.com/sirupsen/logrus/issues/137
|
||||
data[k] = v.Error()
|
||||
default:
|
||||
data[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
serialized, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
|
||||
}
|
||||
return append(serialized, '\n'), nil
|
||||
}
|
||||
|
||||
// log successful test runs
|
||||
func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
|
||||
// calculate the test case duration
|
||||
duration := time.Since(startTime)
|
||||
// log with the fields as per mint
|
||||
fields := log.Fields{"name": "aws-sdk-go", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": PASS}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
// log failed test runs
|
||||
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
|
||||
// calculate the test case duration
|
||||
duration := time.Since(startTime)
|
||||
var fields log.Fields
|
||||
// log with the fields as per mint
|
||||
if err != nil {
|
||||
fields = log.Fields{"name": "aws-sdk-go", "function": function, "args": args,
|
||||
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message, "error": err}
|
||||
} else {
|
||||
fields = log.Fields{"name": "aws-sdk-go", "function": function, "args": args,
|
||||
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message}
|
||||
}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
func randString(n int, src rand.Source, prefix string) string {
|
||||
b := make([]byte, n)
|
||||
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
|
||||
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
|
||||
if remain == 0 {
|
||||
cache, remain = src.Int63(), letterIdxMax
|
||||
}
|
||||
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
|
||||
b[i] = letterBytes[idx]
|
||||
i--
|
||||
}
|
||||
cache >>= letterIdxBits
|
||||
remain--
|
||||
}
|
||||
return prefix + string(b[0:30-len(prefix)])
|
||||
}
|
||||
|
||||
func cleanup(s3Client *s3.S3, bucket string, object string, function string,
|
||||
args map[string]interface{}, startTime time.Time, deleteBucket bool) {
|
||||
|
||||
// Deleting the object, just in case it was created. Will not check for errors.
|
||||
s3Client.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object),
|
||||
})
|
||||
|
||||
if deleteBucket {
|
||||
_, err := s3Client.DeleteBucket(&s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "AWS SDK Go DeleteBucket Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func testPresignedPutInvalidHash(s3Client *s3.S3) {
|
||||
startTime := time.Now()
|
||||
function := "PresignedPut"
|
||||
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-")
|
||||
object := "presignedTest"
|
||||
expiry := 1 * time.Minute
|
||||
args := map[string]interface{}{
|
||||
"bucketName": bucket,
|
||||
"objectName": object,
|
||||
"expiry": expiry,
|
||||
}
|
||||
|
||||
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
defer cleanup(s3Client, bucket, object, function, args, startTime, true)
|
||||
|
||||
req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object),
|
||||
ContentType: aws.String("application/octet-stream"),
|
||||
})
|
||||
|
||||
req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "invalid-sha256")
|
||||
url, err := req.Presign(expiry)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "AWS SDK Go presigned Put request creation failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
rreq, err := http.NewRequest("PUT", url, bytes.NewReader([]byte("")))
|
||||
rreq.Header.Add("X-Amz-Content-Sha256", "invalid-sha256")
|
||||
rreq.Header.Add("Content-Type", "application/octet-stream")
|
||||
|
||||
resp, err := http.DefaultClient.Do(rreq)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "AWS SDK Go presigned put request failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
dec := xml.NewDecoder(resp.Body)
|
||||
errResp := ErrorResponse{}
|
||||
err = dec.Decode(&errResp)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "AWS SDK Go unmarshalling xml failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
if errResp.Code != "XAmzContentSHA256Mismatch" {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go presigned PUT expected to fail with XAmzContentSHA256Mismatch but got %v", errResp.Code), errors.New("AWS S3 error code mismatch")).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
successLogger(function, args, startTime).Info()
|
||||
}
|
||||
|
||||
func testListObjects(s3Client *s3.S3) {
|
||||
startTime := time.Now()
|
||||
function := "testListObjects"
|
||||
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-")
|
||||
object1 := "testObject1"
|
||||
object2 := "testObject2"
|
||||
expiry := 1 * time.Minute
|
||||
args := map[string]interface{}{
|
||||
"bucketName": bucket,
|
||||
"objectName1": object1,
|
||||
"objectName2": object2,
|
||||
"expiry": expiry,
|
||||
}
|
||||
|
||||
getKeys := func(objects []*s3.Object) []string {
|
||||
var rv []string
|
||||
for _, obj := range objects {
|
||||
rv = append(rv, *obj.Key)
|
||||
}
|
||||
return rv
|
||||
}
|
||||
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
defer cleanup(s3Client, bucket, object1, function, args, startTime, true)
|
||||
defer cleanup(s3Client, bucket, object2, function, args, startTime, false)
|
||||
|
||||
listInput := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
MaxKeys: aws.Int64(1000),
|
||||
Prefix: aws.String(""),
|
||||
}
|
||||
result, err := s3Client.ListObjectsV2(listInput)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
if *result.KeyCount != 0 {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 0 key but got %v, %v", result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal()
|
||||
return
|
||||
}
|
||||
putInput1 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")),
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object1),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput1)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
putInput2 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")),
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object2),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput2)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
result, err = s3Client.ListObjectsV2(listInput)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
if *result.KeyCount != 2 {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 2 key but got %v, %v", *result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
successLogger(function, args, startTime).Info()
|
||||
}
|
||||
|
||||
func main() {
|
||||
endpoint := os.Getenv("SERVER_ENDPOINT")
|
||||
accessKey := os.Getenv("ACCESS_KEY")
|
||||
secretKey := os.Getenv("SECRET_KEY")
|
||||
secure := os.Getenv("ENABLE_HTTPS")
|
||||
sdkEndpoint := "http://" + endpoint
|
||||
if secure == "1" {
|
||||
sdkEndpoint = "https://" + endpoint
|
||||
}
|
||||
|
||||
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
|
||||
newSession := session.New()
|
||||
s3Config := &aws.Config{
|
||||
Credentials: creds,
|
||||
Endpoint: aws.String(sdkEndpoint),
|
||||
Region: aws.String("us-east-1"),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
}
|
||||
|
||||
// Create an S3 service object in the default region.
|
||||
s3Client := s3.New(newSession, s3Config)
|
||||
|
||||
// Output to stdout instead of the default stderr
|
||||
log.SetOutput(os.Stdout)
|
||||
// create custom formatter
|
||||
mintFormatter := mintJSONFormatter{}
|
||||
// set custom formatter
|
||||
log.SetFormatter(&mintFormatter)
|
||||
// log Info or above -- success cases are Info level, failures are Fatal level
|
||||
log.SetLevel(log.InfoLevel)
|
||||
// execute tests
|
||||
testPresignedPutInvalidHash(s3Client)
|
||||
testListObjects(s3Client)
|
||||
}
|
28
mint/run/core/aws-sdk-go/run.sh
Executable file
28
mint/run/core/aws-sdk-go/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
/mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file"
|
30
mint/run/core/aws-sdk-java/run.sh
Executable file
30
mint/run/core/aws-sdk-java/run.sh
Executable file
@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2018 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
cd /mint/run/core/aws-sdk-java/ || exit -1
|
||||
|
||||
java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file"
|
19
mint/run/core/aws-sdk-php/README.md
Normal file
19
mint/run/core/aws-sdk-php/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `aws-sdk-php` tests
|
||||
This directory serves as the location for Mint tests using `aws-sdk-php`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests is added into `quick-tests.php` as new functions.
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
6
mint/run/core/aws-sdk-php/composer.json
Normal file
6
mint/run/core/aws-sdk-php/composer.json
Normal file
@ -0,0 +1,6 @@
|
||||
{
|
||||
"require": {
|
||||
"aws/aws-sdk-php": "^3.30",
|
||||
"GuzzleHttp/Psr7": "^1.4"
|
||||
}
|
||||
}
|
1101
mint/run/core/aws-sdk-php/quick-tests.php
Normal file
1101
mint/run/core/aws-sdk-php/quick-tests.php
Normal file
File diff suppressed because it is too large
Load Diff
28
mint/run/core/aws-sdk-php/run.sh
Executable file
28
mint/run/core/aws-sdk-php/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint, (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file"
|
19
mint/run/core/aws-sdk-ruby/README.md
Normal file
19
mint/run/core/aws-sdk-ruby/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `aws-sdk-ruby` tests
|
||||
This directory serves as the location for Mint tests using `aws-sdk-ruby`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests is added into `aws-stub-test.rb` as new functions.
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
855
mint/run/core/aws-sdk-ruby/aws-stub-tests.rb
Executable file
855
mint/run/core/aws-sdk-ruby/aws-stub-tests.rb
Executable file
@ -0,0 +1,855 @@
|
||||
#!/usr/bin/env ruby
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
require 'aws-sdk'
|
||||
require 'securerandom'
|
||||
require 'net/http'
|
||||
require 'multipart_body'
|
||||
|
||||
class AwsSdkRubyTest
|
||||
# Set variables necessary to create an s3 client instance.
|
||||
# Get them from the environment variables
|
||||
|
||||
# Region information, eg. "us-east-1"
|
||||
region = ENV['SERVER_REGION'] ||= 'SERVER_REGION is not set'
|
||||
# Minio server, eg. "play.minio.io:9000"
|
||||
endpoint = ENV['SERVER_ENDPOINT'] ||= 'SERVER_ENDPOINT is not set'
|
||||
access_key_id = ENV['ACCESS_KEY'] ||= 'ACCESS_KEY is not set'
|
||||
secret_access_key = ENV['SECRET_KEY'] ||= 'SECRET_KEY is not set'
|
||||
enable_https = ENV['ENABLE_HTTPS']
|
||||
endpoint = enable_https == '1' ? 'https://' + endpoint : 'http://' + endpoint
|
||||
# Create s3 client instances, "s3Resource" and "s3Client"
|
||||
@@s3 = Aws::S3::Resource.new(region: region,
|
||||
endpoint: endpoint,
|
||||
access_key_id: access_key_id,
|
||||
secret_access_key: secret_access_key,
|
||||
force_path_style: true)
|
||||
|
||||
def initialize_log_output(meth, alert = nil)
|
||||
# Initialize and return log content in log_output hash table
|
||||
|
||||
# Collect args in args_arr
|
||||
args_arr = method(meth).parameters.flatten.map(&:to_s)
|
||||
.reject { |x| x == 'req' || x == 'opt' }
|
||||
# Create and return log output content
|
||||
{ name: 'aws-sdk-ruby',
|
||||
function: "#{meth}(#{args_arr.join(',')})", # method name and arguments
|
||||
args: args_arr, # array of arg names. This'll be replaced with a
|
||||
# a arg/value pairs insdie the caller method
|
||||
duration: 0, # test runtime duration in seconds
|
||||
alert: alert,
|
||||
message: nil,
|
||||
error: nil }
|
||||
end
|
||||
|
||||
def get_random_bucket_name()
|
||||
bucket_name = "aws-sdk-ruby-bucket-"+SecureRandom.hex(6)
|
||||
return bucket_name
|
||||
end
|
||||
|
||||
def calculate_duration(t2, t1)
|
||||
# Durations are in miliseconds, with precision of 2 decimal places
|
||||
((t2 - t1) * 1000).round(2)
|
||||
end
|
||||
|
||||
def print_log(log_output, start_time)
|
||||
# Calculate duration in miliseconds
|
||||
log_output[:duration] = calculate_duration(Time.now, start_time)
|
||||
# Get rid of the log_output fields if nil
|
||||
puts log_output.delete_if{|k, value| value == nil}.to_json
|
||||
# Exit at the first failure
|
||||
exit 1 if log_output[:status] == 'FAIL'
|
||||
end
|
||||
|
||||
def cleanUp(buckets, log_output)
|
||||
# Removes objects and bucket if bucket exists
|
||||
bucket_name = ''
|
||||
buckets.each do |b|
|
||||
bucket_name = b
|
||||
if bucketExistsWrapper(b, log_output)
|
||||
removeObjectsWrapper(b, log_output)
|
||||
removeBucketWrapper(b, log_output)
|
||||
end
|
||||
end
|
||||
rescue => e
|
||||
raise "Failed to clean-up bucket '#{bucket_name}', #{e}"
|
||||
end
|
||||
|
||||
#
|
||||
# API commands/methods
|
||||
#
|
||||
def makeBucket(bucket_name)
|
||||
# Creates a bucket, "bucket_name"
|
||||
# on S3 client , "s3".
|
||||
# Returns bucket_name if already exists
|
||||
@@s3.bucket(bucket_name).exists? ? @@s3.bucket(bucket_name) : @@s3.create_bucket(bucket: bucket_name)
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def makeBucketWrapper(bucket_name, log_output)
|
||||
makeBucket(bucket_name)
|
||||
rescue => e
|
||||
log_output[:function] = "makeBucket(bucket_name)"
|
||||
log_output[:args] = {'bucket_name': bucket_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def removeBucket(bucket_name)
|
||||
# Deletes/removes bucket, "bucket_name" on S3 client, "s3"
|
||||
@@s3.bucket(bucket_name).delete
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def removeBucketWrapper(bucket_name, log_output)
|
||||
removeBucket(bucket_name)
|
||||
rescue => e
|
||||
log_output[:function] = "removeBucket(bucket_name)"
|
||||
log_output[:args] = {'bucket_name': bucket_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def putObject(bucket_name, file)
|
||||
# Creates "file" (full path) in bucket, "bucket_name",
|
||||
# on S3 client, "s3"
|
||||
file_name = File.basename(file)
|
||||
@@s3.bucket(bucket_name).object(file_name).upload_file(file)
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def putObjectWrapper(bucket_name, file, log_output)
|
||||
putObject(bucket_name, file)
|
||||
rescue => e
|
||||
log_output[:function] = "putObject(bucket_name, file)"
|
||||
log_output[:args] = {'bucket_name': bucket_name,
|
||||
'file': file}
|
||||
raise e
|
||||
end
|
||||
|
||||
def getObject(bucket_name, file, destination)
|
||||
# Gets/Downloads file, "file",
|
||||
# from bucket, "bucket_name", of S3 client, "s3"
|
||||
file_name = File.basename(file)
|
||||
dest = File.join(destination, file_name)
|
||||
@@s3.bucket(bucket_name).object(file_name).get(response_target: dest)
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def getObjectWrapper(bucket_name, file, destination, log_output)
|
||||
getObject(bucket_name, file, destination)
|
||||
rescue => e
|
||||
log_output[:function] = "getObject(bucket_name, file)"
|
||||
log_output[:args] = {'bucket_name': bucket_name,
|
||||
'file': file,
|
||||
'destination': destination}
|
||||
raise e
|
||||
end
|
||||
|
||||
def copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '')
|
||||
# Copies file, "file_name", from source bucket,
|
||||
# "source_bucket_name", to target bucket,
|
||||
# "target_bucket_name", on S3 client, "s3"
|
||||
target_file_name = source_file_name if target_file_name.empty?
|
||||
source = @@s3.bucket(source_bucket_name)
|
||||
target = @@s3.bucket(target_bucket_name)
|
||||
source_obj = source.object(source_file_name)
|
||||
target_obj = target.object(target_file_name)
|
||||
source_obj.copy_to(target_obj)
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def copyObjectWrapper(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '', log_output)
|
||||
copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name)
|
||||
rescue => e
|
||||
log_output[:function] = 'copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '')'
|
||||
log_output[:args] = {'source_bucket_name': source_bucket_name,
|
||||
'target_bucket_name': target_bucket_name,
|
||||
'source_file_name': source_file_name,
|
||||
'target_file_name': target_file_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def removeObject(bucket_name, file)
|
||||
# Deletes file in bucket,
|
||||
# "bucket_name", on S3 client, "s3".
|
||||
# If file, "file_name" does not exist,
|
||||
# it quietly returns without any error message
|
||||
@@s3.bucket(bucket_name).object(file).delete
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def removeObjectWrapper(bucket_name, file_name, log_output)
|
||||
removeObject(bucket_name, file_name)
|
||||
rescue => e
|
||||
log_output[:function] = "removeObject(bucket_name, file_name)"
|
||||
log_output[:args] = {'bucket_name': bucket_name,
|
||||
'file_name': file_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def removeObjects(bucket_name)
|
||||
# Deletes all files in bucket, "bucket_name"
|
||||
# on S3 client, "s3"
|
||||
file_name = ''
|
||||
@@s3.bucket(bucket_name).objects.each do |obj|
|
||||
file_name = obj.key
|
||||
obj.delete
|
||||
end
|
||||
rescue => e
|
||||
raise "File name: '#{file_name}', #{e}"
|
||||
end
|
||||
|
||||
def removeObjectsWrapper(bucket_name, log_output)
|
||||
removeObjects(bucket_name)
|
||||
rescue => e
|
||||
log_output[:function] = 'removeObjects(bucket_name)'
|
||||
log_output[:args] = {'bucket_name': bucket_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def listBuckets
|
||||
# Returns an array of bucket names on S3 client, "s3"
|
||||
bucket_name_list = []
|
||||
@@s3.buckets.each do |b|
|
||||
bucket_name_list.push(b.name)
|
||||
end
|
||||
return bucket_name_list
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def listBucketsWrapper(log_output)
|
||||
listBuckets
|
||||
rescue => e
|
||||
log_output[:function] = 'listBuckets'
|
||||
log_output[:args] = {}
|
||||
raise e
|
||||
end
|
||||
|
||||
def listObjects(bucket_name)
|
||||
# Returns an array of object/file names
|
||||
# in bucket, "bucket_name", on S3 client, "s3"
|
||||
object_list = []
|
||||
@@s3.bucket(bucket_name).objects.each do |obj|
|
||||
object_list.push(obj.key)
|
||||
end
|
||||
return object_list
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def listObjectsWrapper(bucket_name, log_output)
|
||||
listObjects(bucket_name)
|
||||
rescue => e
|
||||
log_output[:function] = 'listObjects(bucket_name)'
|
||||
log_output[:args] = {'bucket_name': bucket_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def statObject(bucket_name, file_name)
|
||||
return @@s3.bucket(bucket_name).object(file_name).exists?
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def statObjectWrapper(bucket_name, file_name, log_output)
|
||||
statObject(bucket_name, file_name)
|
||||
rescue => e
|
||||
log_output[:function] = 'statObject(bucket_name, file_name)'
|
||||
log_output[:args] = {'bucket_name': bucket_name,
|
||||
'file_name': file_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def bucketExists?(bucket_name)
|
||||
# Returns true if bucket, "bucket_name", exists,
|
||||
# false otherwise
|
||||
return @@s3.bucket(bucket_name).exists?
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def bucketExistsWrapper(bucket_name, log_output)
|
||||
bucketExists?(bucket_name)
|
||||
rescue => e
|
||||
log_output[:function] = 'bucketExists?(bucket_name)'
|
||||
log_output[:args] = {'bucket_name': bucket_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def presignedGet(bucket_name, file_name)
|
||||
# Returns download/get url
|
||||
obj = @@s3.bucket(bucket_name).object(file_name)
|
||||
return obj.presigned_url(:get, expires_in: 600)
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def presignedGetWrapper(bucket_name, file_name, log_output)
|
||||
presignedGet(bucket_name, file_name)
|
||||
rescue => e
|
||||
log_output[:function] = 'presignedGet(bucket_name, file_name)'
|
||||
log_output[:args] = {'bucket_name': bucket_name,
|
||||
'file_name': file_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def presignedPut(bucket_name, file_name)
|
||||
# Returns put url
|
||||
obj = @@s3.bucket(bucket_name).object(file_name)
|
||||
return obj.presigned_url(:put, expires_in: 600)
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def presignedPutWrapper(bucket_name, file_name, log_output)
|
||||
presignedPut(bucket_name, file_name)
|
||||
rescue => e
|
||||
log_output[:function] = 'presignedPut(bucket_name, file_name)'
|
||||
log_output[:args] = {'bucket_name': bucket_name,
|
||||
'file_name': file_name}
|
||||
raise e
|
||||
end
|
||||
|
||||
def presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)
|
||||
# Returns upload/post url
|
||||
obj = @@s3.bucket(bucket_name).object(file_name)
|
||||
return obj.presigned_post(expires: Time.now + expires_in_sec,
|
||||
content_length_range: 1..max_byte_size)
|
||||
rescue => e
|
||||
raise e
|
||||
end
|
||||
|
||||
def presignedPostWrapper(bucket_name, file_name, expires_in_sec, max_byte_size, log_output)
|
||||
presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)
|
||||
rescue => e
|
||||
log_output[:function] = 'presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)'
|
||||
log_output[:args] = {'bucket_name': bucket_name,
|
||||
'file_name': file_name,
|
||||
'expires_in_sec': expires_in_sec,
|
||||
'max_byte_size': max_byte_size}
|
||||
raise e
|
||||
end
|
||||
|
||||
# To be addressed. S3 API 'get_bucket_policy' does not work!
|
||||
# def getBucketPolicy(bucket_name)
|
||||
# # Returns bucket policy
|
||||
# return @@s3.bucket(bucket_name).get_bucket_policy
|
||||
# rescue => e
|
||||
# raise e
|
||||
# end
|
||||
|
||||
#
|
||||
# Test case methods
|
||||
#
|
||||
def listBucketsTest()
|
||||
# Tests listBuckets api command by creating
|
||||
# new buckets from bucket_name_list
|
||||
|
||||
# get random bucket names and create list
|
||||
bucket_name1 = get_random_bucket_name()
|
||||
bucket_name2 = get_random_bucket_name()
|
||||
bucket_name_list = [bucket_name1, bucket_name2]
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('listBuckets')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
prev_total_buckets = listBucketsWrapper(log_output).length
|
||||
new_buckets = bucket_name_list.length
|
||||
bucket_name_list.each do |b|
|
||||
makeBucketWrapper(b, log_output)
|
||||
end
|
||||
new_total_buckets = prev_total_buckets + new_buckets
|
||||
if new_total_buckets >= prev_total_buckets + new_buckets
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = 'Could not find expected number of buckets'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp(bucket_name_list, log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def makeBucketTest()
|
||||
# Tests makeBucket api command.
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('makeBucket')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
|
||||
if bucketExistsWrapper(bucket_name, log_output)
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = 'Bucket expected to be created does not exist'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def bucketExistsNegativeTest()
|
||||
# Tests bucketExists api command.
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('bucketExists?')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
if !bucketExistsWrapper(bucket_name, log_output)
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = "Failed to return 'false' for a non-existing bucket"
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def removeBucketTest()
|
||||
# Tests removeBucket api command.
|
||||
|
||||
# get a random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('removeBucket')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
removeBucketWrapper(bucket_name, log_output)
|
||||
if !bucketExistsWrapper(bucket_name, log_output)
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = 'Bucket expected to be removed still exists'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def putObjectTest(file)
|
||||
# Tests putObject api command by uploading a file
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('putObject')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
putObjectWrapper(bucket_name, file, log_output)
|
||||
if statObjectWrapper(bucket_name, File.basename(file), log_output)
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = "Status for the created object returned 'false'"
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def removeObjectTest(file)
|
||||
# Tests removeObject api command by uploading and removing a file
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('removeObject')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
putObjectWrapper(bucket_name, file, log_output)
|
||||
removeObjectWrapper(bucket_name, File.basename(file), log_output)
|
||||
if !statObjectWrapper(bucket_name, File.basename(file), log_output)
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = "Status for the removed object returned 'true'"
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def getObjectTest(file, destination)
|
||||
# Tests getObject api command
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('getObject')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
putObjectWrapper(bucket_name, file, log_output)
|
||||
getObjectWrapper(bucket_name, file, destination, log_output)
|
||||
if system("ls -l #{destination} > /dev/null")
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = "Downloaded object does not exist at #{destination}"
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def listObjectsTest(file_list)
|
||||
# Tests listObjects api command
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('listObjects')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
# Put all objects into the bucket
|
||||
file_list.each do |f|
|
||||
putObjectWrapper(bucket_name, f, log_output)
|
||||
end
|
||||
# Total number of files uploaded
|
||||
expected_no = file_list.length
|
||||
# Actual number is what api returns
|
||||
actual_no = listObjectsWrapper(bucket_name, log_output).length
|
||||
# Compare expected and actual values
|
||||
if expected_no == actual_no
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = 'Expected and actual number of listed files/objects do not match!'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def copyObjectTest(data_dir, source_file_name, target_file_name = '')
|
||||
# Tests copyObject api command
|
||||
|
||||
# get random bucket names
|
||||
source_bucket_name = get_random_bucket_name()
|
||||
target_bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('copyObject')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
target_file_name = source_file_name if target_file_name.empty?
|
||||
makeBucketWrapper(source_bucket_name, log_output)
|
||||
makeBucketWrapper(target_bucket_name, log_output)
|
||||
putObjectWrapper(source_bucket_name,
|
||||
File.join(data_dir, source_file_name), log_output)
|
||||
copyObjectWrapper(source_bucket_name, target_bucket_name,
|
||||
source_file_name, target_file_name, log_output)
|
||||
# Check if copy worked fine
|
||||
if statObjectWrapper(target_bucket_name, target_file_name, log_output)
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = 'Copied file could not be found in the expected location'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([source_bucket_name, target_bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def presignedGetObjectTest(data_dir, file_name)
|
||||
# Tests presignedGetObject api command
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('presignedGet')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
file = File.join(data_dir, file_name)
|
||||
# Get check sum value without the file name
|
||||
cksum_orig = `cksum #{file}`.split[0..1]
|
||||
putObjectWrapper(bucket_name, file, log_output)
|
||||
get_url = presignedGetWrapper(bucket_name, file_name, log_output)
|
||||
# Download the file using the URL
|
||||
# generated by presignedGet api command
|
||||
`wget -O /tmp/#{file_name}, '#{get_url}' > /dev/null 2>&1`
|
||||
# Get check sum value for the downloaded file
|
||||
# Split to get rid of the file name
|
||||
cksum_new = `cksum /tmp/#{file_name}`.split[0..1]
|
||||
|
||||
# Check if check sum values for the orig file
|
||||
# and the downloaded file match
|
||||
if cksum_orig == cksum_new
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = 'Check sum values do NOT match'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def presignedPutObjectTest(data_dir, file_name)
|
||||
# Tests presignedPutObject api command
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('presignedPut')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
file = File.join(data_dir, file_name)
|
||||
|
||||
# Get check sum value and
|
||||
# split to get rid of the file name
|
||||
cksum_orig = `cksum #{file}`.split[0..1]
|
||||
|
||||
# Generate presigned Put URL and parse it
|
||||
uri = URI.parse(presignedPutWrapper(bucket_name, file_name, log_output))
|
||||
request = Net::HTTP::Put.new(uri.request_uri, 'x-amz-acl' => 'public-read')
|
||||
request.body = IO.read(File.join(data_dir, file_name))
|
||||
|
||||
http = Net::HTTP.new(uri.host, uri.port)
|
||||
http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1'
|
||||
|
||||
http.request(request)
|
||||
|
||||
if statObjectWrapper(bucket_name, file_name, log_output)
|
||||
getObjectWrapper(bucket_name, file_name, '/tmp', log_output)
|
||||
cksum_new = `cksum /tmp/#{file_name}`.split[0..1]
|
||||
# Check if check sum values of the orig file
|
||||
# and the downloaded file match
|
||||
if cksum_orig == cksum_new
|
||||
log_output[:status] = 'PASS'
|
||||
else
|
||||
log_output[:error] = 'Check sum values do NOT match'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
else
|
||||
log_output[:error] = 'Expected to be created object does NOT exist'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
|
||||
def presignedPostObjectTest(data_dir, file_name,
|
||||
expires_in_sec, max_byte_size)
|
||||
# Tests presignedPostObject api command
|
||||
|
||||
# get random bucket name
|
||||
bucket_name = get_random_bucket_name()
|
||||
# Initialize hash table, 'log_output'
|
||||
log_output = initialize_log_output('presignedPost')
|
||||
# Prepare arg/value hash table and set it in log_output
|
||||
arg_value_hash = {}
|
||||
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
|
||||
log_output[:args] = arg_value_hash
|
||||
|
||||
begin
|
||||
start_time = Time.now
|
||||
makeBucketWrapper(bucket_name, log_output)
|
||||
file = File.join(data_dir, file_name)
|
||||
|
||||
# Get check sum value and split it
|
||||
# into parts to get rid of the file name
|
||||
cksum_orig = `cksum #{file}`.split[0..1]
|
||||
# Create the presigned POST url
|
||||
post = presignedPostWrapper(bucket_name, file_name,
|
||||
expires_in_sec, max_byte_size, log_output)
|
||||
|
||||
# Prepare multi parts array for POST command request
|
||||
file_part = Part.new name: 'file',
|
||||
body: IO.read(File.join(data_dir, file_name)),
|
||||
filename: file_name,
|
||||
content_type: 'application/octet-stream'
|
||||
parts = [file_part]
|
||||
# Add POST fields into parts array
|
||||
post.fields.each do |field, value|
|
||||
parts.push(Part.new(field, value))
|
||||
end
|
||||
boundary = "---------------------------#{rand(10_000_000_000_000_000)}"
|
||||
body_parts = MultipartBody.new parts, boundary
|
||||
|
||||
# Parse presigned Post URL
|
||||
uri = URI.parse(post.url)
|
||||
|
||||
# Create the HTTP objects
|
||||
http = Net::HTTP.new(uri.host, uri.port)
|
||||
http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1'
|
||||
request = Net::HTTP::Post.new(uri.request_uri)
|
||||
request.body = body_parts.to_s
|
||||
request.content_type = "multipart/form-data; boundary=#{boundary}"
|
||||
# Send the request
|
||||
log_output[:error] = http.request(request)
|
||||
|
||||
if statObjectWrapper(bucket_name, file_name, log_output)
|
||||
getObjectWrapper(bucket_name, file_name, '/tmp', log_output)
|
||||
cksum_new = `cksum /tmp/#{file_name}`.split[0..1]
|
||||
# Check if check sum values of the orig file
|
||||
# and the downloaded file match
|
||||
if cksum_orig == cksum_new
|
||||
log_output[:status] = 'PASS'
|
||||
# FIXME: HTTP No Content error, status code=204 is returned as error
|
||||
log_output[:error] = nil
|
||||
else
|
||||
log_output[:error] = 'Check sum values do NOT match'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
else
|
||||
log_output[:error] = 'Expected to be created object does NOT exist'
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
cleanUp([bucket_name], log_output)
|
||||
rescue => log_output[:error]
|
||||
log_output[:status] = 'FAIL'
|
||||
end
|
||||
|
||||
print_log(log_output, start_time)
|
||||
end
|
||||
end
|
||||
|
||||
# MAIN CODE
|
||||
|
||||
# Create test Class instance and call the tests
|
||||
aws = AwsSdkRubyTest.new
|
||||
file_name1 = 'datafile-1-kB'
|
||||
file_new_name = 'datafile-1-kB-copy'
|
||||
file_name_list = ['datafile-1-kB', 'datafile-1-b', 'datafile-6-MB']
|
||||
# Add data_dir in front of each file name in file_name_list
|
||||
# The location where the bucket and file
|
||||
# objects are going to be created.
|
||||
data_dir = ENV['MINT_DATA_DIR'] ||= 'MINT_DATA_DIR is not set'
|
||||
file_list = file_name_list.map { |f| File.join(data_dir, f) }
|
||||
destination = '/tmp'
|
||||
|
||||
aws.listBucketsTest()
|
||||
aws.listObjectsTest(file_list)
|
||||
aws.makeBucketTest()
|
||||
aws.bucketExistsNegativeTest()
|
||||
aws.removeBucketTest()
|
||||
aws.putObjectTest(File.join(data_dir, file_name1))
|
||||
aws.removeObjectTest(File.join(data_dir, file_name1))
|
||||
aws.getObjectTest(File.join(data_dir, file_name1), destination)
|
||||
aws.copyObjectTest(data_dir, file_name1)
|
||||
aws.copyObjectTest(data_dir, file_name1, file_new_name)
|
||||
aws.presignedGetObjectTest(data_dir, file_name1)
|
||||
aws.presignedPutObjectTest(data_dir, file_name1)
|
||||
aws.presignedPostObjectTest(data_dir, file_name1, 60, 3*1024*1024)
|
29
mint/run/core/aws-sdk-ruby/run.sh
Executable file
29
mint/run/core/aws-sdk-ruby/run.sh
Executable file
@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
chmod a+x aws-stub-tests.rb
|
||||
ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file"
|
19
mint/run/core/awscli/README.md
Normal file
19
mint/run/core/awscli/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `awscli` tests
|
||||
This directory serves as the location for Mint tests using `awscli`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests is added into `test.sh` as new functions.
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
51
mint/run/core/awscli/run.sh
Executable file
51
mint/run/core/awscli/run.sh
Executable file
@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# configure awscli
|
||||
aws configure set aws_access_key_id "$ACCESS_KEY"
|
||||
aws configure set aws_secret_access_key "$SECRET_KEY"
|
||||
aws configure set default.region "$SERVER_REGION"
|
||||
|
||||
# run tests for virtual style if provided
|
||||
if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then
|
||||
# Setup endpoint scheme
|
||||
endpoint="http://$DOMAIN:$SERVER_PORT"
|
||||
if [ "$ENABLE_HTTPS" -eq 1 ]; then
|
||||
endpoint="https://$DOMAIN:$SERVER_PORT"
|
||||
fi
|
||||
dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root
|
||||
echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf
|
||||
aws configure set default.s3.addressing_style virtual
|
||||
./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file"
|
||||
aws configure set default.s3.addressing_style path
|
||||
fi
|
||||
|
||||
endpoint="http://$SERVER_ENDPOINT"
|
||||
if [ "$ENABLE_HTTPS" -eq 1 ]; then
|
||||
endpoint="https://$SERVER_ENDPOINT"
|
||||
fi
|
||||
# run path style tests
|
||||
./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file"
|
1411
mint/run/core/awscli/test.sh
Executable file
1411
mint/run/core/awscli/test.sh
Executable file
File diff suppressed because it is too large
Load Diff
195
mint/run/core/healthcheck/healthcheck.go
Normal file
195
mint/run/core/healthcheck/healthcheck.go
Normal file
@ -0,0 +1,195 @@
|
||||
/*
|
||||
*
|
||||
* Mint, (C) 2019 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
pass = "PASS" // Indicate that a test passed
|
||||
fail = "FAIL" // Indicate that a test failed
|
||||
livenessPath = "/minio/health/live"
|
||||
readinessPath = "/minio/health/ready"
|
||||
prometheusPath = "/minio/prometheus/metrics"
|
||||
timeout = time.Duration(30 * time.Second)
|
||||
)
|
||||
|
||||
type mintJSONFormatter struct {
|
||||
}
|
||||
|
||||
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
|
||||
data := make(log.Fields, len(entry.Data))
|
||||
for k, v := range entry.Data {
|
||||
switch v := v.(type) {
|
||||
case error:
|
||||
// Otherwise errors are ignored by `encoding/json`
|
||||
// https://github.com/sirupsen/logrus/issues/137
|
||||
data[k] = v.Error()
|
||||
default:
|
||||
data[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
serialized, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
|
||||
}
|
||||
return append(serialized, '\n'), nil
|
||||
}
|
||||
|
||||
// log successful test runs
|
||||
func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
|
||||
// calculate the test case duration
|
||||
duration := time.Since(startTime)
|
||||
// log with the fields as per mint
|
||||
fields := log.Fields{"name": "healthcheck", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": pass}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
// log failed test runs
|
||||
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
|
||||
// calculate the test case duration
|
||||
duration := time.Since(startTime)
|
||||
var fields log.Fields
|
||||
// log with the fields as per mint
|
||||
if err != nil {
|
||||
fields = log.Fields{"name": "healthcheck", "function": function, "args": args,
|
||||
"duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, "error": err}
|
||||
} else {
|
||||
fields = log.Fields{"name": "healthcheck", "function": function, "args": args,
|
||||
"duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message}
|
||||
}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
func testLivenessEndpoint(endpoint string) {
|
||||
startTime := time.Now()
|
||||
function := "testLivenessEndpoint"
|
||||
|
||||
u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, livenessPath))
|
||||
if err != nil {
|
||||
// Could not parse URL successfully
|
||||
failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Liveness handler failed", err).Fatal()
|
||||
}
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
}
|
||||
client := &http.Client{Transport: tr, Timeout: timeout}
|
||||
resp, err := client.Get(u.String())
|
||||
if err != nil {
|
||||
// GET request errored
|
||||
failureLog(function, nil, startTime, "", "GET request failed", err).Fatal()
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
// Status not 200 OK
|
||||
failureLog(function, nil, startTime, "", "GET /minio/health/live returned non OK status", err).Fatal()
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
defer successLogger(function, nil, startTime).Info()
|
||||
}
|
||||
|
||||
func testReadinessEndpoint(endpoint string) {
|
||||
startTime := time.Now()
|
||||
function := "testReadinessEndpoint"
|
||||
|
||||
u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, readinessPath))
|
||||
if err != nil {
|
||||
// Could not parse URL successfully
|
||||
failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Readiness handler failed", err).Fatal()
|
||||
}
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
}
|
||||
client := &http.Client{Transport: tr, Timeout: timeout}
|
||||
resp, err := client.Get(u.String())
|
||||
if err != nil {
|
||||
// GET request errored
|
||||
failureLog(function, nil, startTime, "", "GET request to Readiness endpoint failed", err).Fatal()
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
// Status not 200 OK
|
||||
failureLog(function, nil, startTime, "", "GET /minio/health/ready returned non OK status", err).Fatal()
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
defer successLogger(function, nil, startTime).Info()
|
||||
}
|
||||
|
||||
func testPrometheusEndpoint(endpoint string) {
|
||||
startTime := time.Now()
|
||||
function := "testPrometheusEndpoint"
|
||||
|
||||
u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, prometheusPath))
|
||||
if err != nil {
|
||||
// Could not parse URL successfully
|
||||
failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Prometheus handler failed", err).Fatal()
|
||||
}
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
}
|
||||
client := &http.Client{Transport: tr, Timeout: timeout}
|
||||
resp, err := client.Get(u.String())
|
||||
if err != nil {
|
||||
// GET request errored
|
||||
failureLog(function, nil, startTime, "", "GET request to Prometheus endpoint failed", err).Fatal()
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
// Status not 200 OK
|
||||
failureLog(function, nil, startTime, "", "GET /minio/prometheus/metrics returned non OK status", err).Fatal()
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
defer successLogger(function, nil, startTime).Info()
|
||||
}
|
||||
|
||||
func main() {
|
||||
endpoint := os.Getenv("SERVER_ENDPOINT")
|
||||
secure := os.Getenv("ENABLE_HTTPS")
|
||||
endpoint = "http://" + endpoint
|
||||
if secure == "1" {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
|
||||
// Output to stdout instead of the default stderr
|
||||
log.SetOutput(os.Stdout)
|
||||
// create custom formatter
|
||||
mintFormatter := mintJSONFormatter{}
|
||||
// set custom formatter
|
||||
log.SetFormatter(&mintFormatter)
|
||||
// log Info or above -- success cases are Info level, failures are Fatal level
|
||||
log.SetLevel(log.InfoLevel)
|
||||
// execute tests
|
||||
testLivenessEndpoint(endpoint)
|
||||
testReadinessEndpoint(endpoint)
|
||||
testPrometheusEndpoint(endpoint)
|
||||
}
|
28
mint/run/core/healthcheck/run.sh
Executable file
28
mint/run/core/healthcheck/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2019 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
/mint/run/core/healthcheck/healthcheck 1>>"$output_log_file" 2>"$error_log_file"
|
19
mint/run/core/mc/README.md
Normal file
19
mint/run/core/mc/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `mc` tests
|
||||
This directory serves as the location for Mint tests using `mc`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests is added into `test.sh` as new functions.
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
27
mint/run/core/mc/run.sh
Executable file
27
mint/run/core/mc/run.sh
Executable file
@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
./functional-tests.sh 1>>"$output_log_file" 2>"$error_log_file"
|
26
mint/run/core/minio-dotnet/run.sh
Executable file
26
mint/run/core/minio-dotnet/run.sh
Executable file
@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
/mint/run/core/minio-dotnet/out/Minio.Functional.Tests 1>>"$output_log_file" 2>"$error_log_file"
|
19
mint/run/core/minio-go/README.md
Normal file
19
mint/run/core/minio-go/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `minio-go` tests
|
||||
This directory serves as the location for Mint tests using `minio-go`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests are added in functional tests of minio-go. Please check https://github.com/minio/minio-go
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
28
mint/run/core/minio-go/run.sh
Executable file
28
mint/run/core/minio-go/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
/mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file"
|
19
mint/run/core/minio-java/README.md
Normal file
19
mint/run/core/minio-java/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `minio-java` tests
|
||||
This directory serves as the location for Mint tests using `minio-java`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests is added in functional tests of minio-java. Please check https://github.com/minio/minio-java
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
34
mint/run/core/minio-java/run.sh
Executable file
34
mint/run/core/minio-java/run.sh
Executable file
@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
endpoint="http://$SERVER_ENDPOINT"
|
||||
if [ "$ENABLE_HTTPS" -eq 1 ]; then
|
||||
endpoint="https://$SERVER_ENDPOINT"
|
||||
fi
|
||||
|
||||
java -Xmx4096m -Xms256m -cp "/mint/run/core/minio-java/*:." FunctionalTest \
|
||||
"$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file"
|
19
mint/run/core/minio-js/README.md
Normal file
19
mint/run/core/minio-js/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `minio-js` tests
|
||||
This directory serves as the location for Mint tests using `minio-js`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests is added in functional tests of minio-js. Please check https://github.com/minio/minio-js
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
69
mint/run/core/minio-js/minioreporter.js
Normal file
69
mint/run/core/minio-js/minioreporter.js
Normal file
@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Minio Reporter for JSON formatted logging, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
var mocha = require('mocha');
|
||||
module.exports = minioreporter;
|
||||
|
||||
function minioreporter(runner) {
|
||||
mocha.reporters.Base.call(this, runner);
|
||||
var self = this;
|
||||
|
||||
runner.on('pass', function (test) {
|
||||
GenerateJsonEntry(test)
|
||||
});
|
||||
|
||||
runner.on('fail', function (test, err) {
|
||||
GenerateJsonEntry(test, err)
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert test result into a JSON object and print on the console.
|
||||
*
|
||||
* @api private
|
||||
* @param test, err
|
||||
*/
|
||||
|
||||
function GenerateJsonEntry (test, err) {
|
||||
var res = test.title.split("_")
|
||||
var jsonEntry = {};
|
||||
|
||||
jsonEntry.name = "minio-js"
|
||||
|
||||
if (res.length > 0 && res[0].length) {
|
||||
jsonEntry.function = res[0]
|
||||
}
|
||||
|
||||
if (res.length > 1 && res[1].length) {
|
||||
jsonEntry.args = res[1]
|
||||
}
|
||||
|
||||
jsonEntry.duration = test.duration
|
||||
|
||||
if (res.length > 2 && res[2].length) {
|
||||
jsonEntry.alert = res[2]
|
||||
}
|
||||
|
||||
if (err != null ) {
|
||||
jsonEntry.status = "FAIL"
|
||||
jsonEntry.error = err.stack.replace(/\n/g, " ").replace(/ +(?= )/g,'')
|
||||
} else {
|
||||
jsonEntry.status = "PASS"
|
||||
}
|
||||
|
||||
process.stdout.write(JSON.stringify(jsonEntry) + "\n")
|
||||
}
|
49
mint/run/core/minio-js/package.json
Normal file
49
mint/run/core/minio-js/package.json
Normal file
@ -0,0 +1,49 @@
|
||||
{
|
||||
"name": "bin",
|
||||
"version": "1.0.0",
|
||||
"main": "functional_test.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"app-module-path": "*",
|
||||
"async": "*",
|
||||
"block-stream2": "*",
|
||||
"concat-stream": "*",
|
||||
"es6-error": "*",
|
||||
"json-stream": "*",
|
||||
"lodash": "*",
|
||||
"mime-types": "*",
|
||||
"mkdirp": "*",
|
||||
"moment": "*",
|
||||
"source-map-support": "*",
|
||||
"through2": "*",
|
||||
"xml": "*",
|
||||
"xml2js": "*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"browserify": "*",
|
||||
"chai": "*",
|
||||
"gulp": "*",
|
||||
"gulp-babel": "*",
|
||||
"gulp-jscs": "*",
|
||||
"jshint":"2.*",
|
||||
"gulp-jshint": "*",
|
||||
"gulp-mocha": "*",
|
||||
"gulp-notify": "*",
|
||||
"gulp-sourcemaps": "*",
|
||||
"jshint-stylish": "*",
|
||||
"mocha": "*",
|
||||
"mocha-steps": "*",
|
||||
"nock": "*",
|
||||
"rewire": "*",
|
||||
"superagent": "*"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "mocha"
|
||||
}
|
||||
}
|
28
mint/run/core/minio-js/run.sh
Executable file
28
mint/run/core/minio-js/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
./node_modules/mocha/bin/mocha -R minioreporter -b --exit 1>>"$output_log_file" 2>"$error_log_file"
|
19
mint/run/core/minio-py/README.md
Normal file
19
mint/run/core/minio-py/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `minio-py` tests
|
||||
This directory serves as the location for Mint tests using `minio-py`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests is added in functional tests of minio-py. Please check https://github.com/minio/minio-py
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
28
mint/run/core/minio-py/run.sh
Executable file
28
mint/run/core/minio-py/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file"
|
2
mint/run/core/s3cmd/.gitignore
vendored
Normal file
2
mint/run/core/s3cmd/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
*~
|
||||
*.log
|
19
mint/run/core/s3cmd/README.md
Normal file
19
mint/run/core/s3cmd/README.md
Normal file
@ -0,0 +1,19 @@
|
||||
## `s3cmd` tests
|
||||
This directory serves as the location for Mint tests using `s3cmd`. Top level `mint.sh` calls `run.sh` to execute tests.
|
||||
|
||||
## Adding new tests
|
||||
New tests is added into `test.sh` as new functions.
|
||||
|
||||
## Running tests manually
|
||||
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
|
||||
- Call `run.sh` with output log file and error log file. for example
|
||||
```bash
|
||||
export MINT_DATA_DIR=~/my-mint-dir
|
||||
export MINT_MODE=core
|
||||
export SERVER_ENDPOINT="play.minio.io:9000"
|
||||
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
export ENABLE_HTTPS=1
|
||||
export SERVER_REGION=us-east-1
|
||||
./run.sh /tmp/output.log /tmp/error.log
|
||||
```
|
28
mint/run/core/s3cmd/run.sh
Executable file
28
mint/run/core/s3cmd/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
./test.sh 1>>"$output_log_file" 2>"$error_log_file"
|
422
mint/run/core/s3cmd/test.sh
Executable file
422
mint/run/core/s3cmd/test.sh
Executable file
@ -0,0 +1,422 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
|
||||
if [ -n "$MINT_MODE" ]; then
|
||||
if [ -z "${MINT_DATA_DIR+x}" ]; then
|
||||
echo "MINT_DATA_DIR not defined"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${SERVER_ENDPOINT+x}" ]; then
|
||||
echo "SERVER_ENDPOINT not defined"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${ACCESS_KEY+x}" ]; then
|
||||
echo "ACCESS_KEY not defined"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${SECRET_KEY+x}" ]; then
|
||||
echo "SECRET_KEY not defined"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "${SERVER_ENDPOINT+x}" ]; then
|
||||
SERVER_ENDPOINT="play.minio.io:9000"
|
||||
ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
||||
SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
||||
ENABLE_HTTPS=1
|
||||
SERVER_REGION="us-east-1"
|
||||
fi
|
||||
|
||||
WORK_DIR="$PWD"
|
||||
DATA_DIR="$MINT_DATA_DIR"
|
||||
if [ -z "$MINT_MODE" ]; then
|
||||
WORK_DIR="$PWD/.run-$RANDOM"
|
||||
DATA_DIR="$WORK_DIR/data"
|
||||
fi
|
||||
|
||||
FILE_1_MB="$DATA_DIR/datafile-1-MB"
|
||||
FILE_65_MB="$DATA_DIR/datafile-65-MB"
|
||||
declare FILE_1_MB_MD5SUM
|
||||
declare FILE_65_MB_MD5SUM
|
||||
|
||||
BUCKET_NAME="s3cmd-test-bucket-$RANDOM"
|
||||
S3CMD=$(which s3cmd)
|
||||
declare -a S3CMD_CMD
|
||||
|
||||
function get_md5sum()
|
||||
{
|
||||
filename="$FILE_1_MB"
|
||||
out=$(md5sum "$filename" 2>/dev/null)
|
||||
rv=$?
|
||||
if [ "$rv" -eq 0 ]; then
|
||||
awk '{ print $1 }' <<< "$out"
|
||||
fi
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function get_time()
|
||||
{
|
||||
date +%s%N
|
||||
}
|
||||
|
||||
function get_duration()
|
||||
{
|
||||
start_time=$1
|
||||
end_time=$(get_time)
|
||||
|
||||
echo $(( (end_time - start_time) / 1000000 ))
|
||||
}
|
||||
|
||||
function log_success()
|
||||
{
|
||||
if [ -n "$MINT_MODE" ]; then
|
||||
printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2"
|
||||
fi
|
||||
}
|
||||
|
||||
function show()
|
||||
{
|
||||
if [ -z "$MINT_MODE" ]; then
|
||||
func_name="$1"
|
||||
echo "Running $func_name()"
|
||||
fi
|
||||
}
|
||||
|
||||
function fail()
|
||||
{
|
||||
rv="$1"
|
||||
shift
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
echo "$@"
|
||||
fi
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function assert()
|
||||
{
|
||||
expected_rv="$1"
|
||||
shift
|
||||
start_time="$1"
|
||||
shift
|
||||
func_name="$1"
|
||||
shift
|
||||
|
||||
err=$("$@" 2>&1)
|
||||
rv=$?
|
||||
if [ "$rv" -ne 0 ] && [ "$expected_rv" -eq 0 ]; then
|
||||
if [ -n "$MINT_MODE" ]; then
|
||||
err=$(printf '%s' "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))')
|
||||
## err is already JSON string, no need to double quote
|
||||
printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err"
|
||||
else
|
||||
echo "s3cmd: $func_name: $err"
|
||||
fi
|
||||
|
||||
exit "$rv"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function assert_success() {
|
||||
assert 0 "$@"
|
||||
}
|
||||
|
||||
function assert_failure() {
|
||||
assert 1 "$@"
|
||||
}
|
||||
|
||||
function s3cmd_cmd()
|
||||
{
|
||||
cmd=( "${S3CMD_CMD[@]}" "$@" )
|
||||
"${cmd[@]}"
|
||||
rv=$?
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function check_md5sum()
|
||||
{
|
||||
expected_checksum="$1"
|
||||
shift
|
||||
filename="$*"
|
||||
|
||||
checksum="$(get_md5sum "$filename")"
|
||||
rv=$?
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
echo "unable to get md5sum for $filename"
|
||||
return "$rv"
|
||||
fi
|
||||
|
||||
if [ "$checksum" != "$expected_checksum" ]; then
|
||||
echo "$filename: md5sum mismatch"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function test_make_bucket()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
bucket_name="s3cmd-test-bucket-$RANDOM"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb "s3://${bucket_name}"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function test_make_bucket_error() {
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
bucket_name="S3CMD-test%bucket%$RANDOM"
|
||||
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function setup()
|
||||
{
|
||||
start_time=$(get_time)
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${BUCKET_NAME}"
|
||||
}
|
||||
|
||||
function teardown()
|
||||
{
|
||||
start_time=$(get_time)
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm --force --recursive "s3://${BUCKET_NAME}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force "s3://${BUCKET_NAME}"
|
||||
}
|
||||
|
||||
function test_put_object()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
object_name="s3cmd-test-object-$RANDOM"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function test_put_object_error()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
start_time=$(get_time)
|
||||
|
||||
object_long_name=$(printf "s3cmd-test-object-%01100d" 1)
|
||||
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_long_name}"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function test_put_object_multipart()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
object_name="s3cmd-test-object-$RANDOM"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function test_get_object()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
object_name="s3cmd-test-object-$RANDOM"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function test_get_object_error()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
object_name="s3cmd-test-object-$RANDOM"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
|
||||
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function test_get_object_multipart()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
object_name="s3cmd-test-object-$RANDOM"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function test_copy_object()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
object_name1="s3cmd-test-object-$RANDOM"
|
||||
object_name2="s3cmd-test-object-$RANDOM"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name1}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd cp "s3://${BUCKET_NAME}/${object_name1}" "s3://${BUCKET_NAME}/${object_name2}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd info "s3://${BUCKET_NAME}/${object_name2}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name1}" "s3://${BUCKET_NAME}/${object_name2}"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function test_sync_list_objects()
|
||||
{
|
||||
show "${FUNCNAME[0]}"
|
||||
|
||||
start_time=$(get_time)
|
||||
bucket_name="s3cmd-test-bucket-$RANDOM"
|
||||
object_name="s3cmd-test-object-$RANDOM"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd sync "$DATA_DIR/" "s3://${bucket_name}"
|
||||
|
||||
diff -bB <(ls "$DATA_DIR") <("${S3CMD_CMD[@]}" ls "s3://${bucket_name}" | awk '{print $4}' | sed "s/s3:*..${bucket_name}.//g") >/dev/null 2>&1
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" fail $? "sync and list differs"
|
||||
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force --recursive "s3://${bucket_name}"
|
||||
|
||||
log_success "$start_time" "${FUNCNAME[0]}"
|
||||
}
|
||||
|
||||
function run_test()
|
||||
{
|
||||
test_make_bucket
|
||||
test_make_bucket_error
|
||||
|
||||
setup
|
||||
|
||||
test_put_object
|
||||
test_put_object_error
|
||||
test_put_object_multipart
|
||||
test_get_object
|
||||
test_get_object_multipart
|
||||
test_copy_object
|
||||
test_sync_list_objects
|
||||
|
||||
teardown
|
||||
}
|
||||
|
||||
function __init__()
|
||||
{
|
||||
set -e
|
||||
|
||||
S3CMD_CONFIG_DIR="/tmp/.s3cmd-$RANDOM"
|
||||
mkdir -p $S3CMD_CONFIG_DIR
|
||||
S3CMD_CONFIG_FILE="$S3CMD_CONFIG_DIR/s3cfg"
|
||||
|
||||
# configure s3cmd
|
||||
cat > $S3CMD_CONFIG_FILE <<EOF
|
||||
signature_v2 = False
|
||||
host_base = $SERVER_ENDPOINT
|
||||
host_bucket = $SERVER_ENDPOINT
|
||||
bucket_location = $SERVER_REGION
|
||||
use_https = $ENABLE_HTTPS
|
||||
access_key = $ACCESS_KEY
|
||||
secret_key = $SECRET_KEY
|
||||
EOF
|
||||
|
||||
# For Mint, setup is already done. For others, setup the environment
|
||||
if [ -z "$MINT_MODE" ]; then
|
||||
mkdir -p "$WORK_DIR"
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
# If s3cmd executable binary is not available in current directory, use it in the path.
|
||||
if [ ! -x "$S3CMD" ]; then
|
||||
echo "'s3cmd' executable binary not found in current directory and in path"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -x "$S3CMD" ]; then
|
||||
echo "$S3CMD executable binary not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
S3CMD_CMD=( "${S3CMD}" --config "$S3CMD_CONFIG_FILE" )
|
||||
|
||||
if [ ! -e "$FILE_1_MB" ]; then
|
||||
shred -n 1 -s 1MB - >"$FILE_1_MB"
|
||||
fi
|
||||
|
||||
if [ ! -e "$FILE_65_MB" ]; then
|
||||
shred -n 1 -s 65MB - >"$FILE_65_MB"
|
||||
fi
|
||||
|
||||
set -E
|
||||
set -o pipefail
|
||||
|
||||
FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")"
|
||||
rv=$?
|
||||
if [ $rv -ne 0 ]; then
|
||||
echo "unable to get md5sum of $FILE_1_MB"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")"
|
||||
rv=$?
|
||||
if [ $rv -ne 0 ]; then
|
||||
echo "unable to get md5sum of $FILE_65_MB"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function main()
|
||||
{
|
||||
( run_test )
|
||||
rv=$?
|
||||
|
||||
rm -fr "$S3CMD_CONFIG_FILE"
|
||||
if [ -z "$MINT_MODE" ]; then
|
||||
rm -fr "$WORK_DIR" "$DATA_DIR"
|
||||
fi
|
||||
|
||||
exit "$rv"
|
||||
}
|
||||
|
||||
__init__ "$@"
|
||||
main "$@"
|
28
mint/run/core/security/run.sh
Executable file
28
mint/run/core/security/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2018 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
/mint/run/core/security/tls-tests 1>>"$output_log_file" 2>"$error_log_file"
|
272
mint/run/core/security/tls-tests.go
Normal file
272
mint/run/core/security/tls-tests.go
Normal file
@ -0,0 +1,272 @@
|
||||
// Mint, (C) 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const testName = "TLS-tests"
|
||||
|
||||
const (
|
||||
// PASS indicate that a test passed
|
||||
PASS = "PASS"
|
||||
// FAIL indicate that a test failed
|
||||
FAIL = "FAIL"
|
||||
// NA indicates that a test is not applicable
|
||||
NA = "NA"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFormatter(&mintJSONFormatter{})
|
||||
log.SetLevel(log.InfoLevel)
|
||||
|
||||
endpoint := os.Getenv("SERVER_ENDPOINT")
|
||||
secure := os.Getenv("ENABLE_HTTPS")
|
||||
if secure != "1" {
|
||||
log.WithFields(log.Fields{"name:": testName, "status": NA, "message": "TLS is not enabled"}).Info()
|
||||
return
|
||||
}
|
||||
|
||||
testTLSVersions(endpoint)
|
||||
testTLSCiphers(endpoint)
|
||||
testTLSEllipticCurves(endpoint)
|
||||
}
|
||||
|
||||
// Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections - fail if so.
|
||||
// Tests whether the endpoint accepts TLS1.2 connections - fail if not.
|
||||
func testTLSVersions(endpoint string) {
|
||||
const function = "TLSVersions"
|
||||
startTime := time.Now()
|
||||
|
||||
// Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections
|
||||
args := map[string]interface{}{
|
||||
"MinVersion": "tls.VersionSSL30",
|
||||
"MaxVersion": "tls.VersionTLS11",
|
||||
}
|
||||
_, err := tls.Dial("tcp", endpoint, &tls.Config{
|
||||
MinVersion: tls.VersionSSL30,
|
||||
MaxVersion: tls.VersionTLS11,
|
||||
})
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", "Endpoint accepts insecure connection", err).Error()
|
||||
return
|
||||
}
|
||||
|
||||
// Tests whether the endpoint accepts TLS1.2 connections
|
||||
args = map[string]interface{}{
|
||||
"MinVersion": "tls.VersionTLS12",
|
||||
}
|
||||
_, err = tls.Dial("tcp", endpoint, &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "Endpoint rejects secure connection", err).Error()
|
||||
return
|
||||
}
|
||||
successLog(function, args, startTime)
|
||||
}
|
||||
|
||||
// Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections - fail if so.
|
||||
// Tests whether the endpoint accepts TLS1.2 connections - fail if not.
|
||||
func testTLSCiphers(endpoint string) {
|
||||
const function = "TLSCiphers"
|
||||
startTime := time.Now()
|
||||
|
||||
// Tests whether the endpoint accepts insecure ciphers
|
||||
args := map[string]interface{}{
|
||||
"MinVersion": "tls.VersionTLS12",
|
||||
"CipherSuites": unsupportedCipherSuites,
|
||||
}
|
||||
_, err := tls.Dial("tcp", endpoint, &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
CipherSuites: unsupportedCipherSuites,
|
||||
})
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", "Endpoint accepts insecure cipher suites", err).Error()
|
||||
return
|
||||
}
|
||||
|
||||
// Tests whether the endpoint accepts at least one secure cipher
|
||||
args = map[string]interface{}{
|
||||
"MinVersion": "tls.VersionTLS12",
|
||||
"CipherSuites": supportedCipherSuites,
|
||||
}
|
||||
_, err = tls.Dial("tcp", endpoint, &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
CipherSuites: supportedCipherSuites,
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "Endpoint rejects all secure cipher suites", err).Error()
|
||||
return
|
||||
}
|
||||
|
||||
// Tests whether the endpoint accepts at least one default cipher
|
||||
args = map[string]interface{}{
|
||||
"MinVersion": "tls.VersionTLS12",
|
||||
"CipherSuites": nil,
|
||||
}
|
||||
_, err = tls.Dial("tcp", endpoint, &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
CipherSuites: nil, // default value
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "Endpoint rejects default cipher suites", err).Error()
|
||||
return
|
||||
}
|
||||
successLog(function, args, startTime)
|
||||
}
|
||||
|
||||
// Tests whether the endpoint accepts the P-384 or P-521 elliptic curve - fail if so.
|
||||
// Tests whether the endpoint accepts Curve25519 or P-256 - fail if not.
|
||||
func testTLSEllipticCurves(endpoint string) {
|
||||
const function = "TLSEllipticCurves"
|
||||
startTime := time.Now()
|
||||
|
||||
// Tests whether the endpoint accepts curves using non-constant time implementations.
|
||||
args := map[string]interface{}{
|
||||
"CurvePreferences": unsupportedCurves,
|
||||
}
|
||||
_, err := tls.Dial("tcp", endpoint, &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
CurvePreferences: unsupportedCurves,
|
||||
CipherSuites: supportedCipherSuites,
|
||||
})
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", "Endpoint accepts insecure elliptic curves", err).Error()
|
||||
return
|
||||
}
|
||||
|
||||
// Tests whether the endpoint accepts curves using constant time implementations.
|
||||
args = map[string]interface{}{
|
||||
"CurvePreferences": unsupportedCurves,
|
||||
}
|
||||
_, err = tls.Dial("tcp", endpoint, &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
CurvePreferences: supportedCurves,
|
||||
CipherSuites: supportedCipherSuites,
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "Endpoint does not accept secure elliptic curves", err).Error()
|
||||
return
|
||||
}
|
||||
successLog(function, args, startTime)
|
||||
}
|
||||
|
||||
func successLog(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
|
||||
duration := time.Since(startTime).Nanoseconds() / 1000000
|
||||
return log.WithFields(log.Fields{
|
||||
"name": testName,
|
||||
"function": function,
|
||||
"args": args,
|
||||
"duration": duration,
|
||||
"status": PASS,
|
||||
})
|
||||
}
|
||||
|
||||
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
|
||||
duration := time.Since(startTime).Nanoseconds() / 1000000
|
||||
fields := log.Fields{
|
||||
"name": testName,
|
||||
"function": function,
|
||||
"args": args,
|
||||
"duration": duration,
|
||||
"status": FAIL,
|
||||
"alert": alert,
|
||||
"message": message,
|
||||
}
|
||||
if err != nil {
|
||||
fields["error"] = err
|
||||
}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
type mintJSONFormatter struct {
|
||||
}
|
||||
|
||||
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
|
||||
data := make(log.Fields, len(entry.Data))
|
||||
for k, v := range entry.Data {
|
||||
switch v := v.(type) {
|
||||
case error:
|
||||
// Otherwise errors are ignored by `encoding/json`
|
||||
// https://github.com/sirupsen/logrus/issues/137
|
||||
data[k] = v.Error()
|
||||
default:
|
||||
data[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
serialized, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
|
||||
}
|
||||
return append(serialized, '\n'), nil
|
||||
}
|
||||
|
||||
// Secure Go implementations of modern TLS ciphers
|
||||
// The following ciphers are excluded because:
|
||||
// - RC4 ciphers: RC4 is broken
|
||||
// - 3DES ciphers: Because of the 64 bit blocksize of DES (Sweet32)
|
||||
// - CBC-SHA256 ciphers: No countermeasures against Lucky13 timing attack
|
||||
// - CBC-SHA ciphers: Legacy ciphers (SHA-1) and non-constant time
|
||||
// implementation of CBC.
|
||||
// (CBC-SHA ciphers can be enabled again if required)
|
||||
// - RSA key exchange ciphers: Disabled because of dangerous PKCS1-v1.5 RSA
|
||||
// padding scheme. See Bleichenbacher attacks.
|
||||
var supportedCipherSuites = []uint16{
|
||||
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
|
||||
}
|
||||
|
||||
// Supported elliptic curves: Implementations are constant-time.
|
||||
var supportedCurves = []tls.CurveID{tls.X25519, tls.CurveP256}
|
||||
|
||||
var unsupportedCipherSuites = []uint16{
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
|
||||
tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, // Broken cipher
|
||||
tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
|
||||
tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, // Broken cipher
|
||||
|
||||
// all RSA-PKCS1-v1.5 ciphers are disabled - danger of Bleichenbacher attack variants
|
||||
tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32
|
||||
tls.TLS_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
|
||||
tls.TLS_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks
|
||||
tls.TLS_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
|
||||
tls.TLS_RSA_WITH_RC4_128_SHA, // Broken cipher
|
||||
|
||||
tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure.
|
||||
tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure.
|
||||
}
|
||||
|
||||
// Unsupported elliptic curves: Implementations are not constant-time.
|
||||
var unsupportedCurves = []tls.CurveID{tls.CurveP384, tls.CurveP521}
|
409
mint/run/core/worm/quick-worm-tests.go
Normal file
409
mint/run/core/worm/quick-worm-tests.go
Normal file
@ -0,0 +1,409 @@
|
||||
/*
|
||||
* Mint, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
|
||||
var randSource *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
const (
|
||||
letterIdxBits = 6 // 6 bits to represent a letter index
|
||||
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
|
||||
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
|
||||
)
|
||||
const (
|
||||
PASS = "PASS" // Indicate that a test passed
|
||||
FAIL = "FAIL" // Indicate that a test failed
|
||||
NA = "NA" // Indicate that a test is not applicable
|
||||
maxPartSize = int64(512 * 1000 * 1024)
|
||||
maxRetries = 1
|
||||
)
|
||||
|
||||
type mintJSONFormatter struct {
|
||||
}
|
||||
|
||||
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
|
||||
data := make(log.Fields, len(entry.Data))
|
||||
for k, v := range entry.Data {
|
||||
switch v := v.(type) {
|
||||
case error:
|
||||
// Otherwise errors are ignored by `encoding/json`
|
||||
// https://github.com/sirupsen/logrus/issues/137
|
||||
data[k] = v.Error()
|
||||
default:
|
||||
data[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
serialized, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
|
||||
}
|
||||
return append(serialized, '\n'), nil
|
||||
}
|
||||
|
||||
// log successful test runs
|
||||
func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
|
||||
// calculate the test case duration
|
||||
duration := time.Since(startTime)
|
||||
// log with the fields as per mint
|
||||
fields := log.Fields{"name": "test worm mode", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": PASS}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
// log failed test runs
|
||||
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
|
||||
// calculate the test case duration
|
||||
duration := time.Since(startTime)
|
||||
var fields log.Fields
|
||||
// log with the fields as per mint
|
||||
if err != nil {
|
||||
fields = log.Fields{"name": "test worm mode", "function": function, "args": args,
|
||||
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message, "error": err}
|
||||
} else {
|
||||
fields = log.Fields{"name": "test worm mode", "function": function, "args": args,
|
||||
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message}
|
||||
}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
func randBucketName() string {
|
||||
b := make([]byte, 55)
|
||||
for i := range b {
|
||||
b[i] = charset[randSource.Intn(len(charset))]
|
||||
}
|
||||
return "bucket-" + string(b)
|
||||
}
|
||||
|
||||
func testPutDeletObject(s3Client *s3.S3) {
|
||||
startTime := time.Now()
|
||||
object := "testObject"
|
||||
function := "PutAndDelete"
|
||||
bucket := randBucketName()
|
||||
expiry := 1 * time.Minute
|
||||
args := map[string]interface{}{
|
||||
"bucketName": bucket,
|
||||
"objectName": object,
|
||||
"expiry": expiry,
|
||||
}
|
||||
// First time bucket creation will be successful
|
||||
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON - CreateBucket Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
// First time put object will be successful
|
||||
putInput1 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("fileToUpload")),
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput1)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON - expected to pass but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
// Put Object
|
||||
putInput2 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("filetouploadSecondTime")),
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput2)
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON Put is expected to fail, but it passed %v", nil), nil).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Deleting the Object
|
||||
delObject := &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
_, err = s3Client.DeleteObject(delObject)
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON Delete is expected to fail, but it passed %v", nil), nil).Fatal()
|
||||
return
|
||||
}
|
||||
successLogger(function, args, startTime).Info()
|
||||
|
||||
}
|
||||
|
||||
func testCopyObject(s3Client *s3.S3) {
|
||||
startTime := time.Now()
|
||||
function := "CopyObject"
|
||||
object := "DestinationObject"
|
||||
object1 := "SourceObject"
|
||||
destinationBucket := randBucketName()
|
||||
sourceBucket := randBucketName()
|
||||
expiry := 1 * time.Minute
|
||||
args := map[string]interface{}{
|
||||
"bucketName": destinationBucket,
|
||||
"objectName": object,
|
||||
"expiry": expiry,
|
||||
}
|
||||
// Create Destination bucket
|
||||
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(destinationBucket),
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON Destination Bucket Creation Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Put object on Destination bucket
|
||||
putInput1 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("file to Upload In Destination")),
|
||||
Bucket: aws.String(destinationBucket),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput1)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON PUT expected to pass but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Create Source bucket
|
||||
_, err1 := s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(sourceBucket),
|
||||
})
|
||||
if err1 != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON Source Bucket Creation Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Put object on Destination bucket
|
||||
putInput2 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("file content to copy ")),
|
||||
Bucket: aws.String(sourceBucket),
|
||||
Key: aws.String(object1),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput2)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON PUT expected to pass but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Test for Copy Object
|
||||
copyInput := &s3.CopyObjectInput{
|
||||
Bucket: aws.String(destinationBucket),
|
||||
CopySource: aws.String(sourceBucket + "/" + object1),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
|
||||
_, err = s3Client.CopyObject(copyInput)
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON Copy Object should fail, but it passed %v", nil), nil).Fatal()
|
||||
return
|
||||
}
|
||||
successLogger(function, args, startTime).Info()
|
||||
|
||||
}
|
||||
|
||||
func testPutMultipart(s3Client *s3.S3) {
|
||||
bucket := randBucketName()
|
||||
startTime := time.Now()
|
||||
object := "testObject"
|
||||
expiry := 1 * time.Minute
|
||||
args := map[string]interface{}{
|
||||
"bucketName": bucket,
|
||||
"objectName": object,
|
||||
"expiry": expiry,
|
||||
}
|
||||
function := "PutMultiPart"
|
||||
file, err := os.Open("/mint/data/datafile-5-MB")
|
||||
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON err opening file", err).Fatal()
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
fileInfo, _ := file.Stat()
|
||||
size := fileInfo.Size()
|
||||
buffer := make([]byte, size)
|
||||
fileType := http.DetectContentType(buffer)
|
||||
file.Read(buffer)
|
||||
|
||||
path := file.Name()
|
||||
input := &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(path),
|
||||
ContentType: aws.String(fileType),
|
||||
}
|
||||
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON Destination Bucket Creation Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
// Upload for the first time
|
||||
resp, err := s3Client.CreateMultipartUpload(input)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON CreateMultipartUpload Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
var curr, partLength int64
|
||||
var remaining = size
|
||||
var completedParts []*s3.CompletedPart
|
||||
partNumber := 1
|
||||
for curr = 0; remaining != 0; curr += partLength {
|
||||
if remaining < maxPartSize {
|
||||
partLength = remaining
|
||||
} else {
|
||||
partLength = maxPartSize
|
||||
}
|
||||
completedPart, err := uploadPart(s3Client, resp, buffer[curr:curr+partLength], partNumber)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON uploadPart Failed", err).Fatal()
|
||||
err := abortMultipartUpload(s3Client, resp)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON abortMultipartUpload Failed", err).Fatal()
|
||||
}
|
||||
return
|
||||
}
|
||||
remaining -= partLength
|
||||
partNumber++
|
||||
completedParts = append(completedParts, completedPart)
|
||||
}
|
||||
_, err = completeMultipartUpload(s3Client, resp, completedParts)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON completeMultipartUpload Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
// These tests should fail
|
||||
_, err = s3Client.CreateMultipartUpload(input)
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON CreateMultipartUpload must fail, but it passed %v", nil), nil).Fatal()
|
||||
return
|
||||
}
|
||||
successLogger(function, args, startTime).Info()
|
||||
|
||||
}
|
||||
func completeMultipartUpload(svc *s3.S3, resp *s3.CreateMultipartUploadOutput, completedParts []*s3.CompletedPart) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
completeInput := &s3.CompleteMultipartUploadInput{
|
||||
Bucket: resp.Bucket,
|
||||
Key: resp.Key,
|
||||
UploadId: resp.UploadId,
|
||||
MultipartUpload: &s3.CompletedMultipartUpload{
|
||||
Parts: completedParts,
|
||||
},
|
||||
}
|
||||
return svc.CompleteMultipartUpload(completeInput)
|
||||
}
|
||||
func uploadPart(svc *s3.S3, resp *s3.CreateMultipartUploadOutput, fileBytes []byte, partNumber int) (*s3.CompletedPart, error) {
|
||||
tryNum := 1
|
||||
partInput := &s3.UploadPartInput{
|
||||
Body: bytes.NewReader(fileBytes),
|
||||
Bucket: resp.Bucket,
|
||||
Key: resp.Key,
|
||||
PartNumber: aws.Int64(int64(partNumber)),
|
||||
UploadId: resp.UploadId,
|
||||
ContentLength: aws.Int64(int64(len(fileBytes))),
|
||||
}
|
||||
|
||||
for tryNum <= maxRetries {
|
||||
uploadResult, err := svc.UploadPart(partInput)
|
||||
if err != nil {
|
||||
if tryNum == maxRetries {
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
return nil, aerr
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
tryNum++
|
||||
} else {
|
||||
return &s3.CompletedPart{
|
||||
ETag: uploadResult.ETag,
|
||||
PartNumber: aws.Int64(int64(partNumber)),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func abortMultipartUpload(svc *s3.S3, resp *s3.CreateMultipartUploadOutput) error {
|
||||
abortInput := &s3.AbortMultipartUploadInput{
|
||||
Bucket: resp.Bucket,
|
||||
Key: resp.Key,
|
||||
UploadId: resp.UploadId,
|
||||
}
|
||||
_, err := svc.AbortMultipartUpload(abortInput)
|
||||
return err
|
||||
}
|
||||
|
||||
func main() {
|
||||
endpoint := os.Getenv("SERVER_ENDPOINT")
|
||||
accessKey := os.Getenv("ACCESS_KEY")
|
||||
secretKey := os.Getenv("SECRET_KEY")
|
||||
secure := os.Getenv("ENABLE_HTTPS")
|
||||
sdkEndpoint := "http://" + endpoint
|
||||
if secure == "1" {
|
||||
sdkEndpoint = "https://" + endpoint
|
||||
}
|
||||
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
|
||||
newSession := session.New()
|
||||
s3Config := &aws.Config{
|
||||
Credentials: creds,
|
||||
Endpoint: aws.String(sdkEndpoint),
|
||||
Region: aws.String("us-east-1"),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
}
|
||||
|
||||
// Create an S3 service object in the default region.
|
||||
s3Client := s3.New(newSession, s3Config)
|
||||
|
||||
// Output to stdout instead of the default stderr
|
||||
log.SetOutput(os.Stdout)
|
||||
// create custom formatter
|
||||
mintFormatter := mintJSONFormatter{}
|
||||
// set custom formatter
|
||||
log.SetFormatter(&mintFormatter)
|
||||
// log Info or above -- success cases are Info level, failures are Fatal level
|
||||
log.SetLevel(log.InfoLevel)
|
||||
// execute tests
|
||||
// Test Put and Delete Object
|
||||
testPutDeletObject(s3Client)
|
||||
//testCopyObject
|
||||
testCopyObject(s3Client)
|
||||
// Test Multipart Upload
|
||||
testPutMultipart(s3Client)
|
||||
}
|
28
mint/run/core/worm/run.sh
Executable file
28
mint/run/core/worm/run.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
/mint/run/core/worm/worm 1>>"$output_log_file" 2>"$error_log_file"
|
Loading…
Reference in New Issue
Block a user