mirror of
https://github.com/minio/minio.git
synced 2025-11-11 06:20:14 -05:00
Automatically set goroutines based on shardSize (#5346)
Update reedsolomon library to enable feature to automatically set number of go-routines based on the input shard size, since shard size is sort of a constant in Minio for objects > 10MiB (default blocksize) klauspost reported around 15-20% improvement in performance numbers on older systems such as AVX and SSE3 ``` name old speed new speed delta Encode10x2x10000-8 5.45GB/s ± 1% 6.22GB/s ± 1% +14.20% (p=0.000 n=9+9) Encode100x20x10000-8 1.44GB/s ± 1% 1.64GB/s ± 1% +13.77% (p=0.000 n=10+10) Encode17x3x1M-8 10.0GB/s ± 5% 12.0GB/s ± 1% +19.88% (p=0.000 n=10+10) Encode10x4x16M-8 7.81GB/s ± 5% 8.56GB/s ± 5% +9.58% (p=0.000 n=10+9) Encode5x2x1M-8 15.3GB/s ± 2% 19.6GB/s ± 2% +28.57% (p=0.000 n=9+10) Encode10x2x1M-8 12.2GB/s ± 5% 15.0GB/s ± 5% +22.45% (p=0.000 n=10+10) Encode10x4x1M-8 7.84GB/s ± 1% 9.03GB/s ± 1% +15.19% (p=0.000 n=9+9) Encode50x20x1M-8 1.73GB/s ± 4% 2.09GB/s ± 4% +20.59% (p=0.000 n=10+9) Encode17x3x16M-8 10.6GB/s ± 1% 11.7GB/s ± 4% +10.12% (p=0.000 n=8+10) ```
This commit is contained in:
committed by
kannappanr
parent
b1fb550d5c
commit
c0721164be
15
vendor/github.com/klauspost/reedsolomon/options.go
generated
vendored
15
vendor/github.com/klauspost/reedsolomon/options.go
generated
vendored
@@ -15,11 +15,12 @@ type options struct {
|
||||
useAVX2, useSSSE3, useSSE2 bool
|
||||
usePAR1Matrix bool
|
||||
useCauchy bool
|
||||
shardSize int
|
||||
}
|
||||
|
||||
var defaultOptions = options{
|
||||
maxGoroutines: 384,
|
||||
minSplitSize: 512,
|
||||
minSplitSize: 1024,
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -46,6 +47,18 @@ func WithMaxGoroutines(n int) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithAutoGoroutines will adjust the number of goroutines for optimal speed with a
|
||||
// specific shard size.
|
||||
// Send in the shard size you expect to send. Other shard sizes will work, but may not
|
||||
// run at the optimal speed.
|
||||
// Overwrites WithMaxGoroutines.
|
||||
// If shardSize <= 0, it is ignored.
|
||||
func WithAutoGoroutines(shardSize int) Option {
|
||||
return func(o *options) {
|
||||
o.shardSize = shardSize
|
||||
}
|
||||
}
|
||||
|
||||
// WithMinSplitSize is the minimum encoding size in bytes per goroutine.
|
||||
// See WithMaxGoroutines on how jobs are split.
|
||||
// If n <= 0, it is ignored.
|
||||
|
||||
Reference in New Issue
Block a user