MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

avoid repeated large allocations for large parts (#17968)

Browse Source 
objects with 10,000 parts and many of them can
cause a large memory spike which can potentially
lead to OOM due to lack of GC.

with previous PR reducing the memory usage significantly
in #17963, this PR reduces this further by 80% under
repeated calls.

Scanner sub-system has no use for the slice of Parts(),
it is better left empty.

```
benchmark                            old ns/op     new ns/op     delta
BenchmarkToFileInfo/ToFileInfo-8     295658        188143        -36.36%

benchmark                            old allocs     new allocs     delta
BenchmarkToFileInfo/ToFileInfo-8     61             60             -1.64%

benchmark                            old bytes     new bytes     delta
BenchmarkToFileInfo/ToFileInfo-8     1097210       227255        -79.29%
```
This commit is contained in:
Harshavardhana
2023-09-02 07:49:24 -07:00
committed by GitHub
parent 8208bcb896
commit 3995355150
9 changed files with 92 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/xl-storage.go
View File

@@ -528,7 +528,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
// Remove filename which is the meta file.
item.transformMetaDir()
fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath())
fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath(), false)
metaDataPoolPut(buf)
if err != nil {
res["err"] = err.Error()
@@ -1437,7 +1437,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
return fi, err
}
fi, err = getFileInfo(buf, volume, path, versionID, readData)
fi, err = getFileInfo(buf, volume, path, versionID, readData, true)
if err != nil {
return fi, err
}
@@ -2410,7 +2410,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
}
// Replace the data of null version or any other existing version-id
ofi, err := xlMeta.ToFileInfo(dstVolume, dstPath, reqVID, false)
ofi, err := xlMeta.ToFileInfo(dstVolume, dstPath, reqVID, false, false)
if err == nil && !ofi.Deleted {
if xlMeta.SharedDataDirCountStr(reqVID, ofi.DataDir) == 0 {
// Purge the destination path as we are not preserving anything