MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

bz2: limit max concurrent CPU (#13458)

Browse Source 
Ensure that bz2 decompression will never take more than 50% CPU.
This commit is contained in:
Klaus Post
2021-10-18 08:44:36 -07:00
committed by GitHub
parent feabd0430c
commit c2eb60df4a
4 changed files with 16 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/s3select/progress.go
View File

@@ -38,6 +38,9 @@ type countUpReader struct {
bytesRead int64
}
// Max bzip2 concurrency across calls. 50% of GOMAXPROCS.
var bz2Limiter = pbzip2.CreateConcurrencyPool((runtime.GOMAXPROCS(0) + 1) / 2)
func (r *countUpReader) Read(p []byte) (n int, err error) {
n, err = r.reader.Read(p)
atomic.AddInt64(&r.bytesRead, int64(n))
@@ -124,7 +127,10 @@ func newProgressReader(rc io.ReadCloser, compType CompressionType) (*progressRea
pr.closer = gzr
case bzip2Type:
ctx, cancel := context.WithCancel(context.Background())
r = pbzip2.NewReader(ctx, scannedReader, pbzip2.DecompressionOptions(pbzip2.BZConcurrency((runtime.GOMAXPROCS(0)+1)/2)))
r = pbzip2.NewReader(ctx, scannedReader, pbzip2.DecompressionOptions(
pbzip2.BZConcurrency((runtime.GOMAXPROCS(0)+1)/2),
pbzip2.BZConcurrencyPool(bz2Limiter),
))
pr.closer = &nopReadCloser{fn: cancel}
case zstdType:
// Set a max window of 64MB. More than reasonable.