avoid pre-populating buffers for deployments < 32GiB memory (#19839)

cmd/common-main.go

@@ -402,6 +402,29 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 		ctxt.certsDirSet = true
 	}
 
+	memAvailable := availableMemory()
+	if ctx.IsSet("memlimit") || ctx.GlobalIsSet("memlimit") {
+		memlimit := ctx.String("memlimit")
+		if memlimit == "" {
+			memlimit = ctx.GlobalString("memlimit")
+		}
+		mlimit, err := humanize.ParseBytes(memlimit)
+		if err != nil {
+			return err
+		}
+		if mlimit > memAvailable {
+			logger.Info("WARNING: maximum memory available (%s) smaller than specified --memlimit=%s, ignoring --memlimit value",
+				humanize.IBytes(memAvailable), memlimit)
+		}
+		ctxt.MemLimit = mlimit
+	} else {
+		ctxt.MemLimit = memAvailable
+	}
+
+	if memAvailable < ctxt.MemLimit {
+		ctxt.MemLimit = memAvailable
+	}
+
 	ctxt.FTP = ctx.StringSlice("ftp")
 	ctxt.SFTP = ctx.StringSlice("sftp")
 	ctxt.Interface = ctx.String("interface")

cmd/erasure-server-pool.go

@@ -104,7 +104,11 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	// Initialize byte pool once for all sets, bpool size is set to
 	// setCount * setDriveCount with each memory upto blockSizeV2.
 	buffers := bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
-	buffers.Populate()
+	if n >= 16384 {
+		// pre-populate buffers only n >= 16384 which is (32Gi/2Mi)
+		// for all setups smaller than this avoid pre-alloc.
+		buffers.Populate()
+	}
 	globalBytePoolCap.Store(buffers)
 
 	var localDrives []StorageAPI

cmd/globals.go

@@ -160,6 +160,8 @@ type serverCtxt struct {
 	FTP  []string
 	SFTP []string
 
+	MemLimit uint64
+
 	UserTimeout         time.Duration
 	ShutdownTimeout     time.Duration
 	IdleTimeout         time.Duration

cmd/handler-api.go

@@ -91,11 +91,11 @@ func availableMemory() (available uint64) {
 	available = 2048 * blockSizeV2 * 2 // Default to 4 GiB when we can't find the limits.
 
 	if runtime.GOOS == "linux" {
-		// Useful in container mode
+		// Honor cgroup limits if set.
 		limit := cgroupMemLimit()
 		if limit > 0 {
-			// A valid value is found, return its 75%
-			available = (limit * 3) / 4
+			// A valid value is found, return its 90%
+			available = (limit * 9) / 10
 			return
 		}
 	} // for all other platforms limits are based on virtual memory.
@@ -104,8 +104,9 @@ func availableMemory() (available uint64) {
 	if err != nil {
 		return
 	}
-	// A valid value is available return its 75%
-	available = (memStats.Available * 3) / 4
+
+	// A valid value is available return its 90%
+	available = (memStats.Available * 9) / 10
 	return
 }
 
@@ -129,7 +130,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 		maxSetDrives := slices.Max(setDriveCounts)
 
 		// Returns 75% of max memory allowed
-		maxMem := availableMemory()
+		maxMem := globalServerCtxt.MemLimit
 
 		// max requests per node is calculated as
 		// total_ram / ram_per_request

cmd/server-main.go

@@ -834,7 +834,7 @@ func serverMain(ctx *cli.Context) {
 
 	// Set system resources to maximum.
 	bootstrapTrace("setMaxResources", func() {
-		_ = setMaxResources(ctx)
+		_ = setMaxResources(globalServerCtxt)
 	})
 
 	// Verify kernel release and version.

cmd/server-rlimit.go

@@ -22,7 +22,6 @@ import (
 	"runtime/debug"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/cli"
 	"github.com/minio/madmin-go/v3/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/sys"
@@ -45,7 +44,7 @@ func oldLinux() bool {
 	return currentKernel < kernel.Version(4, 0, 0)
 }
 
-func setMaxResources(ctx *cli.Context) (err error) {
+func setMaxResources(ctx serverCtxt) (err error) {
 	// Set the Go runtime max threads threshold to 90% of kernel setting.
 	sysMaxThreads, err := sys.GetMaxThreads()
 	if err == nil {
@@ -72,32 +71,28 @@ func setMaxResources(ctx *cli.Context) (err error) {
 		return err
 	}
 
-	// Set max memory limit as current memory limit.
-	if _, maxLimit, err = sys.GetMaxMemoryLimit(); err != nil {
+	_, vssLimit, err := sys.GetMaxMemoryLimit()
+	if err != nil {
 		return err
 	}
 
-	// set debug memory limit instead of GOMEMLIMIT env
-	_ = setDebugMemoryLimit(ctx)
-
-	err = sys.SetMaxMemoryLimit(maxLimit, maxLimit)
-	return err
-}
-
-func setDebugMemoryLimit(ctx *cli.Context) error {
-	if ctx == nil {
-		return nil
+	if vssLimit > 0 && vssLimit < humanize.GiByte {
+		logger.Info("WARNING: maximum virtual memory limit (%s) is too small for 'go runtime', please consider setting `ulimit -v` to unlimited",
+			humanize.IBytes(vssLimit))
 	}
-	if ctx.IsSet("memlimit") {
-		memlimit := ctx.String("memlimit")
-		if memlimit == "" {
-			memlimit = ctx.GlobalString("memlimit")
-		}
-		mlimit, err := humanize.ParseBytes(memlimit)
-		if err != nil {
-			return err
-		}
-		debug.SetMemoryLimit(int64(mlimit))
+
+	if ctx.MemLimit > 0 {
+		maxLimit = ctx.MemLimit
 	}
+
+	if maxLimit > 0 {
+		debug.SetMemoryLimit(int64(maxLimit))
+	}
+
+	// Do not use RLIMIT_AS as that is not useful and at times on systems < 4Gi
+	// this can crash the Go runtime if the value is smaller refer
+	// - https://github.com/golang/go/issues/38010
+	// - https://github.com/golang/go/issues/43699
+	// So do not add `sys.SetMaxMemoryLimit()` this is not useful for any practical purposes.
 	return nil
 }

cmd/test-utils_test.go

@@ -106,7 +106,7 @@ func TestMain(m *testing.M) {
 	// logger.AddTarget(console.New())
 
 	// Set system resources to maximum.
-	setMaxResources(nil)
+	setMaxResources(serverCtxt{})
 
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background(), io.Discard)