MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: under FanOut API avoid repeated md5sum calculation (#17572)

Browse Source 
md5sum calculation has a high CPU overhead, avoid calculating
it repeatedly for similar fanOut calls.

To fix following CPU profiler result
```
(pprof) top10
Showing nodes accounting for 678.68s, 84.67% of 801.54s total
Dropped 1072 nodes (cum <= 4.01s)
Showing top 10 nodes out of 156
      flat  flat%   sum%        cum   cum%
   332.54s 41.49% 41.49%    332.54s 41.49%  runtime/internal/syscall.Syscall6
   228.39s 28.49% 69.98%    228.39s 28.49%  crypto/md5.block
    48.07s  6.00% 75.98%     48.07s  6.00%  runtime.memmove
    28.91s  3.61% 79.59%     28.91s  3.61%  github.com/minio/highwayhash.updateAVX2
     8.25s  1.03% 80.61%      8.25s  1.03%  runtime.futex
     8.25s  1.03% 81.64%     10.81s  1.35%  runtime.step
     6.99s  0.87% 82.52%     22.35s  2.79%  runtime.pcvalue
     6.67s  0.83% 83.35%     38.90s  4.85%  runtime.mallocgc
     5.77s  0.72% 84.07%     32.61s  4.07%  runtime.gentraceback
     4.84s   0.6% 84.67%     10.49s  1.31%  runtime.lock2
```
This commit is contained in:
Harshavardhana
2023-07-05 03:16:05 -07:00
committed by GitHub
parent f6b48ed02a
commit 0bc34952eb
4 changed files with 60 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
internal/hash/reader_test.go
View File

@@ -19,6 +19,7 @@ package hash
import (
"bytes"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
@@ -37,14 +38,15 @@ func TestHashReaderHelperMethods(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if r.MD5HexString() != "e2fc714c4727ee9395f324cd2e7f331f" {
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", r.MD5HexString())
md5sum := r.MD5Current()
if hex.EncodeToString(md5sum) != "e2fc714c4727ee9395f324cd2e7f331f" {
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", hex.EncodeToString(md5sum))
}
if r.SHA256HexString() != "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" {
t.Errorf("Expected sha256hex \"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589\", got %s", r.SHA256HexString())
}
if r.MD5Base64String() != "4vxxTEcn7pOV8yTNLn8zHw==" {
t.Errorf("Expected md5base64 \"4vxxTEcn7pOV8yTNLn8zHw==\", got \"%s\"", r.MD5Base64String())
if base64.StdEncoding.EncodeToString(md5sum) != "4vxxTEcn7pOV8yTNLn8zHw==" {
t.Errorf("Expected md5base64 \"4vxxTEcn7pOV8yTNLn8zHw==\", got \"%s\"", base64.StdEncoding.EncodeToString(md5sum))
}
if r.Size() != 4 {
t.Errorf("Expected size 4, got %d", r.Size())
@@ -56,9 +58,6 @@ func TestHashReaderHelperMethods(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(r.MD5(), expectedMD5) {
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", r.MD5HexString())
}
if !bytes.Equal(r.MD5Current(), expectedMD5) {
t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", hex.EncodeToString(r.MD5Current()))
}