mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
fix: under FanOut API avoid repeated md5sum calculation (#17572)
md5sum calculation has a high CPU overhead, avoid calculating
it repeatedly for similar fanOut calls.
To fix following CPU profiler result
```
(pprof) top10
Showing nodes accounting for 678.68s, 84.67% of 801.54s total
Dropped 1072 nodes (cum <= 4.01s)
Showing top 10 nodes out of 156
flat flat% sum% cum cum%
332.54s 41.49% 41.49% 332.54s 41.49% runtime/internal/syscall.Syscall6
228.39s 28.49% 69.98% 228.39s 28.49% crypto/md5.block
48.07s 6.00% 75.98% 48.07s 6.00% runtime.memmove
28.91s 3.61% 79.59% 28.91s 3.61% github.com/minio/highwayhash.updateAVX2
8.25s 1.03% 80.61% 8.25s 1.03% runtime.futex
8.25s 1.03% 81.64% 10.81s 1.35% runtime.step
6.99s 0.87% 82.52% 22.35s 2.79% runtime.pcvalue
6.67s 0.83% 83.35% 38.90s 4.85% runtime.mallocgc
5.77s 0.72% 84.07% 32.61s 4.07% runtime.gentraceback
4.84s 0.6% 84.67% 10.49s 1.31% runtime.lock2
```
This commit is contained in:
@@ -20,7 +20,9 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
@@ -1290,15 +1292,20 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
buf := bytebufferpool.Get()
|
||||
defer bytebufferpool.Put(buf)
|
||||
|
||||
md5w := md5.New()
|
||||
|
||||
// Maximum allowed fan-out object size.
|
||||
const maxFanOutSize = 16 << 20
|
||||
|
||||
n, err := io.Copy(buf, ioutil.HardLimitReader(pReader, maxFanOutSize))
|
||||
n, err := io.Copy(io.MultiWriter(buf, md5w), ioutil.HardLimitReader(pReader, maxFanOutSize))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Set the correct hex md5sum for the fan-out stream.
|
||||
fanOutOpts.MD5Hex = hex.EncodeToString(md5w.Sum(nil))
|
||||
|
||||
concurrentSize := 100
|
||||
if runtime.GOMAXPROCS(0) < concurrentSize {
|
||||
concurrentSize = runtime.GOMAXPROCS(0)
|
||||
|
||||
Reference in New Issue
Block a user