From 03767d26da701bb9ccbd9814c820cca34c6e51e5 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 19 Apr 2024 04:26:59 -0700 Subject: [PATCH] fix: get rid of large buffers (#19549) these lead to run-away usage of memory beyond which the Go's GC can handle, we have to re-visit this differently, remove this for now. --- cmd/xl-storage.go | 8 -------- internal/ioutil/ioutil.go | 18 ++---------------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 6f229ad0c..044ea2832 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -2124,14 +2124,6 @@ func (s *xlStorage) writeAllDirect(ctx context.Context, filePath string, fileSiz var bufp *[]byte switch { - case fileSize > 0 && fileSize >= xioutil.XXLargeBlock*2: - // use a larger 8MiB buffer for a really really large streamsx. - bufp = xioutil.ODirectPoolXXLarge.Get().(*[]byte) - defer xioutil.ODirectPoolXXLarge.Put(bufp) - case fileSize > 0 && fileSize >= xioutil.XLargeBlock: - // use a larger 4MiB buffer for a really large streams. - bufp = xioutil.ODirectPoolXLarge.Get().(*[]byte) - defer xioutil.ODirectPoolXLarge.Put(bufp) case fileSize <= xioutil.SmallBlock: bufp = xioutil.ODirectPoolSmall.Get().(*[]byte) defer xioutil.ODirectPoolSmall.Put(bufp) diff --git a/internal/ioutil/ioutil.go b/internal/ioutil/ioutil.go index 99fcf1d21..ced8afcfc 100644 --- a/internal/ioutil/ioutil.go +++ b/internal/ioutil/ioutil.go @@ -34,26 +34,12 @@ import ( // Block sizes constant. const ( - SmallBlock = 32 * humanize.KiByte // Default r/w block size for smaller objects. - LargeBlock = 1 * humanize.MiByte // Default r/w block size for normal objects. - XLargeBlock = 4 * humanize.MiByte // Default r/w block size for very large objects. - XXLargeBlock = 8 * humanize.MiByte // Default r/w block size for very very large objects. + SmallBlock = 32 * humanize.KiByte // Default r/w block size for smaller objects. + LargeBlock = 1 * humanize.MiByte // Default r/w block size for normal objects. ) // aligned sync.Pool's var ( - ODirectPoolXXLarge = sync.Pool{ - New: func() interface{} { - b := disk.AlignedBlock(XXLargeBlock) - return &b - }, - } - ODirectPoolXLarge = sync.Pool{ - New: func() interface{} { - b := disk.AlignedBlock(XLargeBlock) - return &b - }, - } ODirectPoolLarge = sync.Pool{ New: func() interface{} { b := disk.AlignedBlock(LargeBlock)