MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-23 12:43:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

debug/s3-verify: simplify the tool to use lower memory footprint (#15110)

Browse Source
This commit is contained in:
Harshavardhana 2022-06-20 10:45:35 -07:00 committed by GitHub
parent 73733a8fb9
commit e83e947ca3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 78 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
docs/debugging/s3-verify/main.go
View File

@ -23,6 +23,7 @@ import (
"crypto/sha256"
"flag"
"fmt"
"hash"
"io"
"log"
"net/url"
@ -133,13 +134,11 @@ func main() {
sopts := minio.ListObjectsOptions{
Recursive: true,
Prefix: sourcePrefix,
WithMetadata: true,
}
topts := minio.ListObjectsOptions{
Recursive: true,
Prefix: targetPrefix,
WithMetadata: true,
}
srcCh := sclnt.ListObjects(context.Background(), sourceBucket, sopts)
@ -150,7 +149,13 @@ func main() {
var srcEOF, tgtEOF bool
srcSha256 := sha256.New()
tgtSha256 := sha256.New()
for {
srcSha256.Reset()
tgtSha256.Reset()
srcEOF = !srcOk
tgtEOF = !tgtOk
@ -182,74 +187,7 @@ func main() {
}
if srcCtnt.Key == tgtCtnt.Key {
var allgood bool
if srcCtnt.Size != tgtCtnt.Size {
fmt.Printf("differ in size sourceSize: %d, targetSize: %d\n", srcCtnt.Size, tgtCtnt.Size)
} else if srcCtnt.ContentType != tgtCtnt.ContentType {
fmt.Printf("differ in contentType source: %s, target: %s\n", srcCtnt.ContentType, tgtCtnt.ContentType)
} else {
opts := minio.GetObjectOptions{}
sobj, err := sclnt.GetObject(context.Background(), sourceBucket, srcCtnt.Key, opts)
if err == nil {
tobj, err := tclnt.GetObject(context.Background(), targetBucket, tgtCtnt.Key, opts)
if err != nil {
fmt.Printf("unreadable on target: %s (%s)\n", tgtCtnt.Key, err)
} else {
srcSha256 := sha256.New()
tgtSha256 := sha256.New()
var sourceFailed, targetFailed bool
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
srcSize, err := io.Copy(srcSha256, sobj)
if err != nil {
fmt.Printf("unreadable on source: %s (%s)\n", srcCtnt.Key, err)
sourceFailed = true
return
}
if srcSize != srcCtnt.Size {
fmt.Printf("unreadable on source - size differs upon read: %s\n", srcCtnt.Key)
sourceFailed = true
}
}()
go func() {
defer wg.Done()
tgtSize, err := io.Copy(tgtSha256, tobj)
if err != nil {
fmt.Printf("unreadable on target: %s (%s)\n", tgtCtnt.Key, err)
targetFailed = true
return
}
if tgtSize != tgtCtnt.Size {
fmt.Printf("unreadable on target - size differs upon read: %s\n", tgtCtnt.Key)
targetFailed = true
}
}()
wg.Wait()
tobj.Close()
if !sourceFailed && !targetFailed {
ssum := srcSha256.Sum(nil)
tsum := tgtSha256.Sum(nil)
if !bytes.Equal(ssum, tsum) {
fmt.Printf("sha256 sum mismatch: %s -> Expected(%x), Found(%x)\n", srcCtnt.Key, ssum, tsum)
} else {
allgood = true
}
}
}
sobj.Close()
} else {
fmt.Printf("unreadable on source: %s (%s)\n", srcCtnt.Key, err)
}
}
if allgood {
if verifyChecksum(sclnt, srcSha256, tgtSha256, srcCtnt, tgtCtnt) {
fmt.Printf("all readable source and target: %s -> %s\n", srcCtnt.Key, tgtCtnt.Key)
}
@ -262,3 +200,69 @@ func main() {
tgtCtnt, tgtOk = <-tgtCh
}
}
func verifyChecksum(sclnt *minio.Client, srcSha256, tgtSha256 hash.Hash, srcCtnt, tgtCtnt minio.ObjectInfo) (allgood bool) {
opts := minio.GetObjectOptions{}
if srcCtnt.Size != tgtCtnt.Size {
fmt.Printf("differ in size sourceSize: %d, targetSize: %d\n", srcCtnt.Size, tgtCtnt.Size)
return false
} else if srcCtnt.ContentType != tgtCtnt.ContentType {
fmt.Printf("differ in contentType source: %s, target: %s\n", srcCtnt.ContentType, tgtCtnt.ContentType)
return false
}
core := minio.Core{Client: sclnt}
sobj, _, _, err := core.GetObject(context.Background(), sourceBucket, srcCtnt.Key, opts)
if err != nil {
fmt.Printf("unreadable on source: %s (%s)\n", srcCtnt.Key, err)
return false
}
tobj, _, _, err := core.GetObject(context.Background(), targetBucket, tgtCtnt.Key, opts)
if err != nil {
fmt.Printf("unreadable on target: %s (%s)\n", tgtCtnt.Key, err)
return false
}
var sourceFailed, targetFailed bool
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
srcSize, err := io.Copy(srcSha256, sobj)
if err != nil {
fmt.Printf("unreadable on source: %s (%s)\n", srcCtnt.Key, err)
sourceFailed = true
} else if srcSize != srcCtnt.Size {
fmt.Printf("unreadable on source - size differs upon read: %s\n", srcCtnt.Key)
sourceFailed = true
}
}()
go func() {
defer wg.Done()
tgtSize, err := io.Copy(tgtSha256, tobj)
if err != nil {
fmt.Printf("unreadable on target: %s (%s)\n", tgtCtnt.Key, err)
targetFailed = true
} else if tgtSize != tgtCtnt.Size {
fmt.Printf("unreadable on target - size differs upon read: %s\n", tgtCtnt.Key)
targetFailed = true
}
}()
wg.Wait()
sobj.Close()
tobj.Close()
if !sourceFailed && !targetFailed {
ssum := srcSha256.Sum(nil)
tsum := tgtSha256.Sum(nil)
allgood = bytes.Equal(ssum, tsum)
if !allgood {
fmt.Printf("sha256 sum mismatch: %s -> Expected(%x), Found(%x)\n", srcCtnt.Key, ssum, tsum)
}
}
return allgood
}