avoid repeated large allocations for large parts (#17968)

objects with 10,000 parts and many of them can
cause a large memory spike which can potentially
lead to OOM due to lack of GC.

with previous PR reducing the memory usage significantly
in #17963, this PR reduces this further by 80% under
repeated calls.

Scanner sub-system has no use for the slice of Parts(),
it is better left empty.

```
benchmark                            old ns/op     new ns/op     delta
BenchmarkToFileInfo/ToFileInfo-8     295658        188143        -36.36%

benchmark                            old allocs     new allocs     delta
BenchmarkToFileInfo/ToFileInfo-8     61             60             -1.64%

benchmark                            old bytes     new bytes     delta
BenchmarkToFileInfo/ToFileInfo-8     1097210       227255        -79.29%
```
This commit is contained in:
Harshavardhana
2023-09-02 07:49:24 -07:00
committed by GitHub
parent 8208bcb896
commit 3995355150
9 changed files with 92 additions and 51 deletions

View File

@@ -1049,7 +1049,7 @@ func TestXMinIOHealingSkip(t *testing.T) {
failOnErr(xl.AddVersion(fi))
var err error
fi, err = xl.ToFileInfo(fi.Volume, fi.Name, fi.VersionID, false)
fi, err = xl.ToFileInfo(fi.Volume, fi.Name, fi.VersionID, false, true)
if err != nil {
t.Fatalf("xl.ToFileInfo failed with %v", err)
}
@@ -1058,3 +1058,41 @@ func TestXMinIOHealingSkip(t *testing.T) {
t.Fatal("Expected fi.Healing to be false")
}
}
func benchmarkManyPartsOptionally(b *testing.B, allParts bool) {
f, err := os.Open("testdata/xl-many-parts.meta")
if err != nil {
b.Fatal(err)
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
b.Fatal(err)
}
buf, _, _ := isIndexedMetaV2(data)
if buf == nil {
b.Fatal("buf == nil")
}
b.Run("ToFileInfo", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_, err = buf.ToFileInfo("volume", "path", "", allParts)
if err != nil {
b.Fatal(err)
}
}
})
}
func BenchmarkToFileInfoNoParts(b *testing.B) {
benchmarkManyPartsOptionally(b, false)
}
func BenchmarkToFileInfoWithParts(b *testing.B) {
benchmarkManyPartsOptionally(b, true)
}