diff --git a/.github/workflows/replication.yaml b/.github/workflows/replication.yaml index 808773325..5e88ae552 100644 --- a/.github/workflows/replication.yaml +++ b/.github/workflows/replication.yaml @@ -37,5 +37,4 @@ jobs: run: | sudo sysctl net.ipv6.conf.all.disable_ipv6=0 sudo sysctl net.ipv6.conf.default.disable_ipv6=0 - go install -v github.com/minio/mc@latest make test-replication diff --git a/.gitignore b/.gitignore index 84a13f436..0475d042e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,10 @@ prime/ stage/ .sia_temp/ config.json -node_modules/ \ No newline at end of file +node_modules/ +mc.* +s3-check-md5* +xl-meta* +healing-bin* +inspect* +200M* diff --git a/docs/bucket/replication/setup_3site_replication.sh b/docs/bucket/replication/setup_3site_replication.sh index 7c07b326e..acf15c93e 100755 --- a/docs/bucket/replication/setup_3site_replication.sh +++ b/docs/bucket/replication/setup_3site_replication.sh @@ -28,8 +28,19 @@ set -e export MINIO_BROWSER=off export MINIO_ROOT_USER="minio" export MINIO_ROOT_PASSWORD="minio123" +export MINIO_KMS_AUTO_ENCRYPTION=off export MINIO_PROMETHEUS_AUTH_TYPE=public -export PATH=${GOPATH}/bin:${PATH} +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +go build ./docs/debugging/s3-check-md5/ +wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \ + && chmod +x mc +wget -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z \ + && chmod +x mc.RELEASE.2021-03-12T03-36-59Z minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & @@ -48,172 +59,199 @@ minio server --address 127.0.0.1:9006 "http://127.0.0.1:9005/tmp/multisitec/data sleep 30 -mc alias set sitea http://127.0.0.1:9001 minio minio123 -mc mb sitea/bucket -mc version enable sitea/bucket -mc mb -l sitea/olockbucket +export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001 +export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9004 +export MC_HOST_sitec=http://minio:minio123@127.0.0.1:9006 -mc alias set siteb http://127.0.0.1:9004 minio minio123 -mc mb siteb/bucket/ -mc version enable siteb/bucket/ -mc mb -l siteb/olockbucket/ +./mc mb sitea/bucket +./mc version enable sitea/bucket +./mc mb -l sitea/olockbucket -mc alias set sitec http://127.0.0.1:9006 minio minio123 -mc mb sitec/bucket/ -mc version enable sitec/bucket/ -mc mb -l sitec/olockbucket +./mc mb siteb/bucket/ +./mc version enable siteb/bucket/ +./mc mb -l siteb/olockbucket/ + +./mc mb sitec/bucket/ +./mc version enable sitec/bucket/ +./mc mb -l sitec/olockbucket echo "adding replication config for site a -> site b" -remote_arn=$(mc admin bucket remote add sitea/bucket/ \ +remote_arn=$(./mc admin bucket remote add sitea/bucket/ \ http://minio:minio123@127.0.0.1:9004/bucket \ --service "replication" --json | jq -r ".RemoteARN") echo "adding replication rule for a -> b : ${remote_arn}" sleep 1 -mc replicate add sitea/bucket/ \ +./mc replicate add sitea/bucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" sleep 1 echo "adding replication config for site b -> site a" -remote_arn=$(mc admin bucket remote add siteb/bucket/ \ +remote_arn=$(./mc admin bucket remote add siteb/bucket/ \ http://minio:minio123@127.0.0.1:9001/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for b -> a : ${remote_arn}" -mc replicate add siteb/bucket/ \ +./mc replicate add siteb/bucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" sleep 1 echo "adding replication config for site a -> site c" -remote_arn=$(mc admin bucket remote add sitea/bucket/ \ +remote_arn=$(./mc admin bucket remote add sitea/bucket/ \ http://minio:minio123@127.0.0.1:9006/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for a -> c : ${remote_arn}" -mc replicate add sitea/bucket/ \ +./mc replicate add sitea/bucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 sleep 1 echo "adding replication config for site c -> site a" -remote_arn=$(mc admin bucket remote add sitec/bucket/ \ +remote_arn=$(./mc admin bucket remote add sitec/bucket/ \ http://minio:minio123@127.0.0.1:9001/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for c -> a : ${remote_arn}" -mc replicate add sitec/bucket/ \ +./mc replicate add sitec/bucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 sleep 1 echo "adding replication config for site b -> site c" -remote_arn=$(mc admin bucket remote add siteb/bucket/ \ +remote_arn=$(./mc admin bucket remote add siteb/bucket/ \ http://minio:minio123@127.0.0.1:9006/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for b -> c : ${remote_arn}" -mc replicate add siteb/bucket/ \ +./mc replicate add siteb/bucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 sleep 1 echo "adding replication config for site c -> site b" -remote_arn=$(mc admin bucket remote add sitec/bucket \ +remote_arn=$(././mc admin bucket remote add sitec/bucket \ http://minio:minio123@127.0.0.1:9004/bucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for c -> b : ${remote_arn}" -mc replicate add sitec/bucket/ \ +./mc replicate add sitec/bucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 sleep 1 echo "adding replication config for olockbucket site a -> site b" -remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \ +remote_arn=$(././mc admin bucket remote add sitea/olockbucket/ \ http://minio:minio123@127.0.0.1:9004/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket a -> b : ${remote_arn}" -mc replicate add sitea/olockbucket/ \ +./mc replicate add sitea/olockbucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" sleep 1 echo "adding replication config for site b -> site a" -remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \ +remote_arn=$(././mc admin bucket remote add siteb/olockbucket/ \ http://minio:minio123@127.0.0.1:9001/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket b -> a : ${remote_arn}" -mc replicate add siteb/olockbucket/ \ +./mc replicate add siteb/olockbucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" sleep 1 echo "adding replication config for olockbucket site a -> site c" -remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \ +remote_arn=$(././mc admin bucket remote add sitea/olockbucket/ \ http://minio:minio123@127.0.0.1:9006/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket a -> c : ${remote_arn}" -mc replicate add sitea/olockbucket/ \ +./mc replicate add sitea/olockbucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 sleep 1 echo "adding replication config for site c -> site a" -remote_arn=$(mc admin bucket remote add sitec/olockbucket/ \ +remote_arn=$(././mc admin bucket remote add sitec/olockbucket/ \ http://minio:minio123@127.0.0.1:9001/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket c -> a : ${remote_arn}" -mc replicate add sitec/olockbucket/ \ +./mc replicate add sitec/olockbucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 sleep 1 echo "adding replication config for site b -> site c" -remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \ +remote_arn=$(././mc admin bucket remote add siteb/olockbucket/ \ http://minio:minio123@127.0.0.1:9006/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket b -> c : ${remote_arn}" -mc replicate add siteb/olockbucket/ \ +./mc replicate add siteb/olockbucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 sleep 1 echo "adding replication config for site c -> site b" -remote_arn=$(mc admin bucket remote add sitec/olockbucket \ +remote_arn=$(././mc admin bucket remote add sitec/olockbucket \ http://minio:minio123@127.0.0.1:9004/olockbucket \ --service "replication" --json | jq -r ".RemoteARN") sleep 1 echo "adding replication rule for olockbucket c -> b : ${remote_arn}" -mc replicate add sitec/olockbucket/ \ +./mc replicate add sitec/olockbucket/ \ --remote-bucket "${remote_arn}" \ --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 sleep 1 echo "Set default governance retention 30d" -mc retention set --default governance 30d sitea/olockbucket +./mc retention set --default governance 30d sitea/olockbucket echo "Copying data to source sitea/bucket" -mc cp --quiet /etc/hosts sitea/bucket +./mc cp --encrypt "sitea/" --quiet /etc/hosts sitea/bucket sleep 1 echo "Copying data to source sitea/olockbucket" -mc cp --quiet /etc/hosts sitea/olockbucket +./mc cp --quiet /etc/hosts sitea/olockbucket sleep 1 echo "Verifying the metadata difference between source and target" -if diff -pruN <(mc stat --json sitea/bucket/hosts | jq .) <(mc stat --json siteb/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then +if diff -pruN <(./mc stat --json sitea/bucket/hosts | jq .) <(./mc stat --json siteb/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then echo "verified sitea-> COMPLETED, siteb-> REPLICA" fi -if diff -pruN <(mc stat --json sitea/bucket/hosts | jq .) <(mc stat --json sitec/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then +if diff -pruN <(./mc stat --json sitea/bucket/hosts | jq .) <(./mc stat --json sitec/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then echo "verified sitea-> COMPLETED, sitec-> REPLICA" fi echo "Verifying the metadata difference between source and target" -if diff -pruN <(mc stat --json sitea/olockbucket/hosts | jq .) <(mc stat --json siteb/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then +if diff -pruN <(./mc stat --json sitea/olockbucket/hosts | jq .) <(./mc stat --json siteb/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then echo "verified sitea-> COMPLETED, siteb-> REPLICA" fi -if diff -pruN <(mc stat --json sitea/olockbucket/hosts | jq .) <(mc stat --json sitec/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then +if diff -pruN <(./mc stat --json sitea/olockbucket/hosts | jq .) <(./mc stat --json sitec/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then echo "verified sitea-> COMPLETED, sitec-> REPLICA" fi +sleep 5 + +head -c 221227088 200M +./mc.RELEASE.2021-03-12T03-36-59Z cp --config-dir ~/.mc --encrypt "sitea" --quiet 200M "sitea/bucket/200M-enc-v1" +./mc.RELEASE.2021-03-12T03-36-59Z cp --config-dir ~/.mc --quiet 200M "sitea/bucket/200M-v1" + +./mc cp --encrypt "sitea" --quiet 200M "sitea/bucket/200M-enc-v2" +./mc cp --quiet 200M "sitea/bucket/200M-v2" + +sleep 10 + +echo "Verifying ETag for all objects" +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket bucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket bucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket bucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket bucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket bucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket bucket + +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket olockbucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket olockbucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket olockbucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket olockbucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket olockbucket +./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket olockbucket + catch diff --git a/docs/debugging/s3-check-md5/main.go b/docs/debugging/s3-check-md5/main.go new file mode 100644 index 000000000..43ebfe565 --- /dev/null +++ b/docs/debugging/s3-check-md5/main.go @@ -0,0 +1,174 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "context" + "crypto/md5" + "flag" + "fmt" + "io" + "log" + "net/url" + "strconv" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + endpoint, accessKey, secretKey string + bucket, prefix string +) + +// getMD5Sum returns MD5 sum of given data. +func getMD5Sum(data []byte) []byte { + hash := md5.New() + hash.Write(data) + return hash.Sum(nil) +} + +func main() { + flag.StringVar(&endpoint, "endpoint", "https://play.min.io", "S3 endpoint URL") + flag.StringVar(&accessKey, "access-key", "Q3AM3UQ867SPQQA43P2F", "S3 Access Key") + flag.StringVar(&secretKey, "secret-key", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "S3 Secret Key") + flag.StringVar(&bucket, "bucket", "", "Select a specific bucket") + flag.StringVar(&prefix, "prefix", "", "Select a prefix") + flag.Parse() + + if endpoint == "" { + log.Fatalln("Endpoint is not provided") + } + + if accessKey == "" { + log.Fatalln("Access key is not provided") + } + + if secretKey == "" { + log.Fatalln("Secret key is not provided") + } + + if bucket == "" && prefix != "" { + log.Fatalln("--prefix is specified without --bucket.") + } + + u, err := url.Parse(endpoint) + if err != nil { + log.Fatalln(err) + } + + s3Client, err := minio.New(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: strings.EqualFold(u.Scheme, "https"), + }) + if err != nil { + log.Fatalln() + } + + // s3Client.TraceOn(os.Stderr) + + var buckets []string + if bucket != "" { + buckets = append(buckets, bucket) + } else { + bucketsInfo, err := s3Client.ListBuckets(context.Background()) + if err != nil { + log.Fatalln(err) + } + for _, b := range bucketsInfo { + buckets = append(buckets, b.Name) + } + } + + for _, bucket := range buckets { + opts := minio.ListObjectsOptions{ + Recursive: true, + Prefix: prefix, + WithVersions: true, + } + + // List all objects from a bucket-name with a matching prefix. + for object := range s3Client.ListObjects(context.Background(), bucket, opts) { + if object.Err != nil { + log.Fatalln("LIST error:", object.Err) + continue + } + if object.IsDeleteMarker { + continue + } + parts := 1 + s := strings.Split(object.ETag, "-") + switch len(s) { + case 1: + // nothing to do + case 2: + if p, err := strconv.Atoi(s[1]); err == nil { + parts = p + } else { + log.Fatalln("ETAG: wrong format:", err) + continue + } + default: + log.Fatalln("Unexpected ETAG format", object.ETag) + } + + var partsMD5Sum [][]byte + + for p := 1; p <= parts; p++ { + obj, err := s3Client.GetObject(context.Background(), bucket, object.Key, + minio.GetObjectOptions{VersionID: object.VersionID, PartNumber: p}) + if err != nil { + log.Fatalln("GET", bucket, object.Key, object.VersionID, "=>", err) + continue + } + h := md5.New() + if _, err := io.Copy(h, obj); err != nil { + log.Fatalln("MD5 calculation error:", bucket, object.Key, object.VersionID, "=>", err) + continue + } + partsMD5Sum = append(partsMD5Sum, h.Sum(nil)) + } + + corrupted := false + + switch parts { + case 1: + md5sum := fmt.Sprintf("%x", partsMD5Sum[0]) + if md5sum != object.ETag { + corrupted = true + } + default: + var totalMD5SumBytes []byte + for _, sum := range partsMD5Sum { + totalMD5SumBytes = append(totalMD5SumBytes, sum...) + } + s3MD5 := fmt.Sprintf("%x-%d", getMD5Sum(totalMD5SumBytes), parts) + if s3MD5 != object.ETag { + corrupted = true + } + } + + if corrupted { + log.Fatalln("CORRUPTED object:", bucket, object.Key, object.VersionID) + } + + log.Println("INTACT", bucket, object.Key, object.VersionID) + } + } +} diff --git a/go.mod b/go.mod index 592383c84..1b593f1ab 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/minio/kes v0.14.0 github.com/minio/madmin-go v1.1.16 github.com/minio/mc v0.0.0-20211118223026-df75eed32e9e // indirect - github.com/minio/minio-go/v7 v7.0.16 + github.com/minio/minio-go/v7 v7.0.17-0.20211201162519-091751848a69 github.com/minio/parquet-go v1.1.0 github.com/minio/pkg v1.1.9 github.com/minio/selfupdate v0.3.1 diff --git a/go.sum b/go.sum index 9f0c09350..f10f3c0a7 100644 --- a/go.sum +++ b/go.sum @@ -1102,8 +1102,8 @@ github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh github.com/minio/minio-go/v7 v7.0.15-0.20211004160302-3b57c1e369ca/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= github.com/minio/minio-go/v7 v7.0.15/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= github.com/minio/minio-go/v7 v7.0.16-0.20211108161804-a7a36ee131df/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= -github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4= -github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= +github.com/minio/minio-go/v7 v7.0.17-0.20211201162519-091751848a69 h1:md7zokRq/+81oc0XAwa451Q/1dE+3QKPOCN6o0iwtbg= +github.com/minio/minio-go/v7 v7.0.17-0.20211201162519-091751848a69/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7 h1:dkfuMNslMjGoJ4ArAMSoQhidYNdm3SgzLBP+f96O3/E= github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7/go.mod h1:lDpuz8nwsfhKlfiBaA3Z8AW019fWEAjO2gltfLbdorE= github.com/minio/operator/logsearchapi v0.0.0-20211011212245-31460bbbc4b7 h1:vFtQqCt67ETp0JAkOKRWTKkgwFv14Vc1jJSxmQ8wJE0=