mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
avoid repeated large allocations for large parts (#17968)
objects with 10,000 parts and many of them can cause a large memory spike which can potentially lead to OOM due to lack of GC. with previous PR reducing the memory usage significantly in #17963, this PR reduces this further by 80% under repeated calls. Scanner sub-system has no use for the slice of Parts(), it is better left empty. ``` benchmark old ns/op new ns/op delta BenchmarkToFileInfo/ToFileInfo-8 295658 188143 -36.36% benchmark old allocs new allocs delta BenchmarkToFileInfo/ToFileInfo-8 61 60 -1.64% benchmark old bytes new bytes delta BenchmarkToFileInfo/ToFileInfo-8 1097210 227255 -79.29% ```
This commit is contained in:
@@ -1049,7 +1049,7 @@ func TestXMinIOHealingSkip(t *testing.T) {
|
||||
failOnErr(xl.AddVersion(fi))
|
||||
|
||||
var err error
|
||||
fi, err = xl.ToFileInfo(fi.Volume, fi.Name, fi.VersionID, false)
|
||||
fi, err = xl.ToFileInfo(fi.Volume, fi.Name, fi.VersionID, false, true)
|
||||
if err != nil {
|
||||
t.Fatalf("xl.ToFileInfo failed with %v", err)
|
||||
}
|
||||
@@ -1058,3 +1058,41 @@ func TestXMinIOHealingSkip(t *testing.T) {
|
||||
t.Fatal("Expected fi.Healing to be false")
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkManyPartsOptionally(b *testing.B, allParts bool) {
|
||||
f, err := os.Open("testdata/xl-many-parts.meta")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
buf, _, _ := isIndexedMetaV2(data)
|
||||
if buf == nil {
|
||||
b.Fatal("buf == nil")
|
||||
}
|
||||
|
||||
b.Run("ToFileInfo", func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err = buf.ToFileInfo("volume", "path", "", allParts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkToFileInfoNoParts(b *testing.B) {
|
||||
benchmarkManyPartsOptionally(b, false)
|
||||
}
|
||||
|
||||
func BenchmarkToFileInfoWithParts(b *testing.B) {
|
||||
benchmarkManyPartsOptionally(b, true)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user