MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-04-01 10:13:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix batch snowball to close channel after listing finishes (#19316)

Browse Source 
panic seen due to premature closing of slow channel while listing is still sending or
list has already closed on the sender's side:
```
panic: close of closed channel

goroutine 13666 [running]:
github.com/minio/minio/internal/ioutil.SafeClose[...](0x101ff51e4?)
	/Users/kp/code/src/github.com/minio/minio/internal/ioutil/ioutil.go:425 +0x24
github.com/minio/minio/cmd.(*erasureServerPools).Walk.func1()
	/Users/kp/code/src/github.com/minio/minio/cmd/erasure-server-pool.go:2142 +0x170
created by github.com/minio/minio/cmd.(*erasureServerPools).Walk in goroutine 1189
	/Users/kp/code/src/github.com/minio/minio/cmd/erasure-server-pool.go:1985 +0x228
```
This commit is contained in:
Poorna 2024-03-21 16:13:43 -07:00 committed by GitHub
parent da81c6cc27
commit 7fd76dbbb7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 18 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
cmd/batch-handlers.go
View File

@ -1064,8 +1064,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() { if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
go func() { go func() {
defer xioutil.SafeClose(slowCh)
// Snowball currently needs the high level minio-go Client, not the Core one // Snowball currently needs the high level minio-go Client, not the Core one
cl, err := miniogo.New(u.Host, &miniogo.Options{ cl, err := miniogo.New(u.Host, &miniogo.Options{
Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken), Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
@ -1081,31 +1079,8 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
// Already validated before arriving here // Already validated before arriving here
smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan) smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
var ( batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
obj = ObjectInfo{} writeFn := func(batch []ObjectInfo) {
batch = make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
valid = true
)
for valid {
obj, valid = <-walkCh
if !valid {
goto write
}
if obj.DeleteMarker || !obj.VersionPurgeStatus.Empty() || obj.Size >= int64(smallerThan) {
slowCh <- obj
continue
}
batch = append(batch, obj)
if len(batch) < *r.Source.Snowball.Batch {
continue
}
write:
if len(batch) > 0 { if len(batch) > 0 {
if err := r.writeAsArchive(ctx, api, cl, batch); err != nil { if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
@ -1118,9 +1093,24 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
// persist in-memory state to disk after every 10secs. // persist in-memory state to disk after every 10secs.
logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job)) logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
} }
batch = batch[:0]
} }
} }
for obj := range walkCh {
if obj.DeleteMarker || !obj.VersionPurgeStatus.Empty() || obj.Size >= int64(smallerThan) {
slowCh <- obj
continue
}
batch = append(batch, obj)
if len(batch) < *r.Source.Snowball.Batch {
continue
}
writeFn(batch)
batch = batch[:0]
}
writeFn(batch)
xioutil.SafeClose(slowCh)
}() }()
} else { } else {
slowCh = walkCh slowCh = walkCh