MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add verification tests for ETag on replicated content (#13857)

This commit is contained in:
Harshavardhana 2021-12-07 10:08:26 -08:00 committed by GitHub
parent b9aae1aaae
commit 92fdcafb66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 266 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/replication.yaml vendored
View File

@ -37,5 +37,4 @@ jobs:
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
go install -v github.com/minio/mc@latest
make test-replication

8
.gitignore vendored
View File

@ -21,4 +21,10 @@ prime/
stage/
.sia_temp/
config.json
node_modules/
node_modules/
mc.*
s3-check-md5*
xl-meta*
healing-bin*
inspect*
200M*

126
docs/bucket/replication/setup_3site_replication.sh
View File

@ -28,8 +28,19 @@ set -e
export MINIO_BROWSER=off
export MINIO_ROOT_USER="minio"
export MINIO_ROOT_PASSWORD="minio123"
export MINIO_KMS_AUTO_ENCRYPTION=off
export MINIO_PROMETHEUS_AUTH_TYPE=public
export PATH=${GOPATH}/bin:${PATH}
export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
unset MINIO_KMS_KES_CERT_FILE
unset MINIO_KMS_KES_KEY_FILE
unset MINIO_KMS_KES_ENDPOINT
unset MINIO_KMS_KES_KEY_NAME
go build ./docs/debugging/s3-check-md5/
wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \
&& chmod +x mc
wget -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z \
&& chmod +x mc.RELEASE.2021-03-12T03-36-59Z
minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
@ -48,172 +59,199 @@ minio server --address 127.0.0.1:9006 "http://127.0.0.1:9005/tmp/multisitec/data
sleep 30
mc alias set sitea http://127.0.0.1:9001 minio minio123
mc mb sitea/bucket
mc version enable sitea/bucket
mc mb -l sitea/olockbucket
export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001
export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9004
export MC_HOST_sitec=http://minio:minio123@127.0.0.1:9006
mc alias set siteb http://127.0.0.1:9004 minio minio123
mc mb siteb/bucket/
mc version enable siteb/bucket/
mc mb -l siteb/olockbucket/
./mc mb sitea/bucket
./mc version enable sitea/bucket
./mc mb -l sitea/olockbucket
mc alias set sitec http://127.0.0.1:9006 minio minio123
mc mb sitec/bucket/
mc version enable sitec/bucket/
mc mb -l sitec/olockbucket
./mc mb siteb/bucket/
./mc version enable siteb/bucket/
./mc mb -l siteb/olockbucket/
./mc mb sitec/bucket/
./mc version enable sitec/bucket/
./mc mb -l sitec/olockbucket
echo "adding replication config for site a -> site b"
remote_arn=$(mc admin bucket remote add sitea/bucket/ \
remote_arn=$(./mc admin bucket remote add sitea/bucket/ \
http://minio:minio123@127.0.0.1:9004/bucket \
--service "replication" --json | jq -r ".RemoteARN")
echo "adding replication rule for a -> b : ${remote_arn}"
sleep 1
mc replicate add sitea/bucket/ \
./mc replicate add sitea/bucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
sleep 1
echo "adding replication config for site b -> site a"
remote_arn=$(mc admin bucket remote add siteb/bucket/ \
remote_arn=$(./mc admin bucket remote add siteb/bucket/ \
http://minio:minio123@127.0.0.1:9001/bucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for b -> a : ${remote_arn}"
mc replicate add siteb/bucket/ \
./mc replicate add siteb/bucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
sleep 1
echo "adding replication config for site a -> site c"
remote_arn=$(mc admin bucket remote add sitea/bucket/ \
remote_arn=$(./mc admin bucket remote add sitea/bucket/ \
http://minio:minio123@127.0.0.1:9006/bucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for a -> c : ${remote_arn}"
mc replicate add sitea/bucket/ \
./mc replicate add sitea/bucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
sleep 1
echo "adding replication config for site c -> site a"
remote_arn=$(mc admin bucket remote add sitec/bucket/ \
remote_arn=$(./mc admin bucket remote add sitec/bucket/ \
http://minio:minio123@127.0.0.1:9001/bucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for c -> a : ${remote_arn}"
mc replicate add sitec/bucket/ \
./mc replicate add sitec/bucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
sleep 1
echo "adding replication config for site b -> site c"
remote_arn=$(mc admin bucket remote add siteb/bucket/ \
remote_arn=$(./mc admin bucket remote add siteb/bucket/ \
http://minio:minio123@127.0.0.1:9006/bucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for b -> c : ${remote_arn}"
mc replicate add siteb/bucket/ \
./mc replicate add siteb/bucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
sleep 1
echo "adding replication config for site c -> site b"
remote_arn=$(mc admin bucket remote add sitec/bucket \
remote_arn=$(././mc admin bucket remote add sitec/bucket \
http://minio:minio123@127.0.0.1:9004/bucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for c -> b : ${remote_arn}"
mc replicate add sitec/bucket/ \
./mc replicate add sitec/bucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
sleep 1
echo "adding replication config for olockbucket site a -> site b"
remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \
remote_arn=$(././mc admin bucket remote add sitea/olockbucket/ \
http://minio:minio123@127.0.0.1:9004/olockbucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for olockbucket a -> b : ${remote_arn}"
mc replicate add sitea/olockbucket/ \
./mc replicate add sitea/olockbucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
sleep 1
echo "adding replication config for site b -> site a"
remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \
remote_arn=$(././mc admin bucket remote add siteb/olockbucket/ \
http://minio:minio123@127.0.0.1:9001/olockbucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for olockbucket b -> a : ${remote_arn}"
mc replicate add siteb/olockbucket/ \
./mc replicate add siteb/olockbucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
sleep 1
echo "adding replication config for olockbucket site a -> site c"
remote_arn=$(mc admin bucket remote add sitea/olockbucket/ \
remote_arn=$(././mc admin bucket remote add sitea/olockbucket/ \
http://minio:minio123@127.0.0.1:9006/olockbucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for olockbucket a -> c : ${remote_arn}"
mc replicate add sitea/olockbucket/ \
./mc replicate add sitea/olockbucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
sleep 1
echo "adding replication config for site c -> site a"
remote_arn=$(mc admin bucket remote add sitec/olockbucket/ \
remote_arn=$(././mc admin bucket remote add sitec/olockbucket/ \
http://minio:minio123@127.0.0.1:9001/olockbucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for olockbucket c -> a : ${remote_arn}"
mc replicate add sitec/olockbucket/ \
./mc replicate add sitec/olockbucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
sleep 1
echo "adding replication config for site b -> site c"
remote_arn=$(mc admin bucket remote add siteb/olockbucket/ \
remote_arn=$(././mc admin bucket remote add siteb/olockbucket/ \
http://minio:minio123@127.0.0.1:9006/olockbucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for olockbucket b -> c : ${remote_arn}"
mc replicate add siteb/olockbucket/ \
./mc replicate add siteb/olockbucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
sleep 1
echo "adding replication config for site c -> site b"
remote_arn=$(mc admin bucket remote add sitec/olockbucket \
remote_arn=$(././mc admin bucket remote add sitec/olockbucket \
http://minio:minio123@127.0.0.1:9004/olockbucket \
--service "replication" --json | jq -r ".RemoteARN")
sleep 1
echo "adding replication rule for olockbucket c -> b : ${remote_arn}"
mc replicate add sitec/olockbucket/ \
./mc replicate add sitec/olockbucket/ \
--remote-bucket "${remote_arn}" \
--replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
sleep 1
echo "Set default governance retention 30d"
mc retention set --default governance 30d sitea/olockbucket
./mc retention set --default governance 30d sitea/olockbucket
echo "Copying data to source sitea/bucket"
mc cp --quiet /etc/hosts sitea/bucket
./mc cp --encrypt "sitea/" --quiet /etc/hosts sitea/bucket
sleep 1
echo "Copying data to source sitea/olockbucket"
mc cp --quiet /etc/hosts sitea/olockbucket
./mc cp --quiet /etc/hosts sitea/olockbucket
sleep 1
echo "Verifying the metadata difference between source and target"
if diff -pruN <(mc stat --json sitea/bucket/hosts | jq .) <(mc stat --json siteb/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then
if diff -pruN <(./mc stat --json sitea/bucket/hosts | jq .) <(./mc stat --json siteb/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then
echo "verified sitea-> COMPLETED, siteb-> REPLICA"
fi
if diff -pruN <(mc stat --json sitea/bucket/hosts | jq .) <(mc stat --json sitec/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then
if diff -pruN <(./mc stat --json sitea/bucket/hosts | jq .) <(./mc stat --json sitec/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then
echo "verified sitea-> COMPLETED, sitec-> REPLICA"
fi
echo "Verifying the metadata difference between source and target"
if diff -pruN <(mc stat --json sitea/olockbucket/hosts | jq .) <(mc stat --json siteb/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then
if diff -pruN <(./mc stat --json sitea/olockbucket/hosts | jq .) <(./mc stat --json siteb/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then
echo "verified sitea-> COMPLETED, siteb-> REPLICA"
fi
if diff -pruN <(mc stat --json sitea/olockbucket/hosts | jq .) <(mc stat --json sitec/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then
if diff -pruN <(./mc stat --json sitea/olockbucket/hosts | jq .) <(./mc stat --json sitec/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then
echo "verified sitea-> COMPLETED, sitec-> REPLICA"
fi
sleep 5
head -c 221227088 </dev/urandom >200M
./mc.RELEASE.2021-03-12T03-36-59Z cp --config-dir ~/.mc --encrypt "sitea" --quiet 200M "sitea/bucket/200M-enc-v1"
./mc.RELEASE.2021-03-12T03-36-59Z cp --config-dir ~/.mc --quiet 200M "sitea/bucket/200M-v1"
./mc cp --encrypt "sitea" --quiet 200M "sitea/bucket/200M-enc-v2"
./mc cp --quiet 200M "sitea/bucket/200M-v2"
sleep 10
echo "Verifying ETag for all objects"
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket bucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket bucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket bucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket bucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket bucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket bucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket olockbucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket olockbucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket olockbucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket olockbucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket olockbucket
./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket olockbucket
catch

174
docs/debugging/s3-check-md5/main.go Normal file
View File

@ -0,0 +1,174 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"crypto/md5"
"flag"
"fmt"
"io"
"log"
"net/url"
"strconv"
"strings"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
var (
endpoint, accessKey, secretKey string
bucket, prefix string
)
// getMD5Sum returns MD5 sum of given data.
func getMD5Sum(data []byte) []byte {
hash := md5.New()
hash.Write(data)
return hash.Sum(nil)
}
func main() {
flag.StringVar(&endpoint, "endpoint", "https://play.min.io", "S3 endpoint URL")
flag.StringVar(&accessKey, "access-key", "Q3AM3UQ867SPQQA43P2F", "S3 Access Key")
flag.StringVar(&secretKey, "secret-key", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "S3 Secret Key")
flag.StringVar(&bucket, "bucket", "", "Select a specific bucket")
flag.StringVar(&prefix, "prefix", "", "Select a prefix")
flag.Parse()
if endpoint == "" {
log.Fatalln("Endpoint is not provided")
}
if accessKey == "" {
log.Fatalln("Access key is not provided")
}
if secretKey == "" {
log.Fatalln("Secret key is not provided")
}
if bucket == "" && prefix != "" {
log.Fatalln("--prefix is specified without --bucket.")
}
u, err := url.Parse(endpoint)
if err != nil {
log.Fatalln(err)
}
s3Client, err := minio.New(u.Host, &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: strings.EqualFold(u.Scheme, "https"),
})
if err != nil {
log.Fatalln()
}
// s3Client.TraceOn(os.Stderr)
var buckets []string
if bucket != "" {
buckets = append(buckets, bucket)
} else {
bucketsInfo, err := s3Client.ListBuckets(context.Background())
if err != nil {
log.Fatalln(err)
}
for _, b := range bucketsInfo {
buckets = append(buckets, b.Name)
}
}
for _, bucket := range buckets {
opts := minio.ListObjectsOptions{
Recursive: true,
Prefix: prefix,
WithVersions: true,
}
// List all objects from a bucket-name with a matching prefix.
for object := range s3Client.ListObjects(context.Background(), bucket, opts) {
if object.Err != nil {
log.Fatalln("LIST error:", object.Err)
continue
}
if object.IsDeleteMarker {
continue
}
parts := 1
s := strings.Split(object.ETag, "-")
switch len(s) {
case 1:
// nothing to do
case 2:
if p, err := strconv.Atoi(s[1]); err == nil {
parts = p
} else {
log.Fatalln("ETAG: wrong format:", err)
continue
}
default:
log.Fatalln("Unexpected ETAG format", object.ETag)
}
var partsMD5Sum [][]byte
for p := 1; p <= parts; p++ {
obj, err := s3Client.GetObject(context.Background(), bucket, object.Key,
minio.GetObjectOptions{VersionID: object.VersionID, PartNumber: p})
if err != nil {
log.Fatalln("GET", bucket, object.Key, object.VersionID, "=>", err)
continue
}
h := md5.New()
if _, err := io.Copy(h, obj); err != nil {
log.Fatalln("MD5 calculation error:", bucket, object.Key, object.VersionID, "=>", err)
continue
}
partsMD5Sum = append(partsMD5Sum, h.Sum(nil))
}
corrupted := false
switch parts {
case 1:
md5sum := fmt.Sprintf("%x", partsMD5Sum[0])
if md5sum != object.ETag {
corrupted = true
}
default:
var totalMD5SumBytes []byte
for _, sum := range partsMD5Sum {
totalMD5SumBytes = append(totalMD5SumBytes, sum...)
}
s3MD5 := fmt.Sprintf("%x-%d", getMD5Sum(totalMD5SumBytes), parts)
if s3MD5 != object.ETag {
corrupted = true
}
}
if corrupted {
log.Fatalln("CORRUPTED object:", bucket, object.Key, object.VersionID)
}
log.Println("INTACT", bucket, object.Key, object.VersionID)
}
}
}

2
go.mod
View File

@ -51,7 +51,7 @@ require (
github.com/minio/kes v0.14.0
github.com/minio/madmin-go v1.1.16
github.com/minio/mc v0.0.0-20211118223026-df75eed32e9e // indirect
github.com/minio/minio-go/v7 v7.0.16
github.com/minio/minio-go/v7 v7.0.17-0.20211201162519-091751848a69
github.com/minio/parquet-go v1.1.0
github.com/minio/pkg v1.1.9
github.com/minio/selfupdate v0.3.1

4
go.sum
View File

@ -1102,8 +1102,8 @@ github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh
github.com/minio/minio-go/v7 v7.0.15-0.20211004160302-3b57c1e369ca/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
github.com/minio/minio-go/v7 v7.0.15/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
github.com/minio/minio-go/v7 v7.0.16-0.20211108161804-a7a36ee131df/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4=
github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
github.com/minio/minio-go/v7 v7.0.17-0.20211201162519-091751848a69 h1:md7zokRq/+81oc0XAwa451Q/1dE+3QKPOCN6o0iwtbg=
github.com/minio/minio-go/v7 v7.0.17-0.20211201162519-091751848a69/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7 h1:dkfuMNslMjGoJ4ArAMSoQhidYNdm3SgzLBP+f96O3/E=
github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7/go.mod h1:lDpuz8nwsfhKlfiBaA3Z8AW019fWEAjO2gltfLbdorE=
github.com/minio/operator/logsearchapi v0.0.0-20211011212245-31460bbbc4b7 h1:vFtQqCt67ETp0JAkOKRWTKkgwFv14Vc1jJSxmQ8wJE0=