warn issues about large block I/O performance for Linux older than 4.0.0 (#14524)

This PR simply adds a warning message when it detects older kernel
versions and warn's them about potential performance issues on this
kernel.

The issue can be seen only with parallel I/O across all drives
on denser setups such as 90 drives or 45 drives per server configurations.

cmd/server-main.go

@@ -448,6 +448,11 @@ func serverMain(ctx *cli.Context) {
 	// Set system resources to maximum.
 	setMaxResources()
 
+	// Verify kernel release and version.
+	if oldLinux() {
+		logger.Info(color.RedBold("WARNING: Detected Linux kernel version older than 4.0.0 release, there are some known potential performance problems with this kernel version. MinIO recommends a minimum of 4.x.x linux kernel version for best performance"))
+	}
+
 	// Configure server.
 	handler, err := configureServerHandler(globalEndpoints)
 	if err != nil {

cmd/server-rlimit.go

@@ -21,10 +21,28 @@ import (
 	"runtime"
 	"runtime/debug"
 
+	"github.com/minio/minio/internal/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sys"
 )
 
+func oldLinux() bool {
+	currentKernel, err := kernel.CurrentVersion()
+	if err != nil {
+		// Could not probe the kernel version
+		return false
+	}
+
+	if currentKernel == 0 {
+		// We could not get any valid value return false
+		return false
+	}
+
+	// legacy linux indicator for printing warnings
+	// about older Linux kernels and Go runtime.
+	return currentKernel < kernel.Version(4, 0, 0)
+}
+
 func setMaxResources() (err error) {
 	// Set the Go runtime max threads threshold to 90% of kernel setting.
 	sysMaxThreads, mErr := sys.GetMaxThreads()

cmd/xl-storage.go

@@ -49,11 +49,12 @@ import (
 
 const (
 	nullVersionID = "null"
-	// Really large streams threshold per shard.
+	// Largest streams threshold per shard.
-	reallyLargeFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
+	largestFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
 
 	// Small file threshold below which data accompanies metadata from storage layer.
 	smallFileThreshold = 128 * humanize.KiByte // Optimized for NVMe/SSDs
+
 	// For hardrives it is possible to set this to a lower value to avoid any
 	// spike in latency. But currently we are simply keeping it optimal for SSDs.
 
@@ -1786,8 +1787,8 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 	}()
 
 	var bufp *[]byte
-	if fileSize > 0 && fileSize >= reallyLargeFileThreshold {
+	if fileSize > 0 && fileSize >= largestFileThreshold {
-		// use a larger 4MiB buffer for really large streams.
+		// use a larger 4MiB buffer for a really large streams.
 		bufp = xioutil.ODirectPoolXLarge.Get().(*[]byte)
 		defer xioutil.ODirectPoolXLarge.Put(bufp)
 	} else {

internal/kernel/kernel.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build linux
+// +build linux
+
+package kernel
+
+import (
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+var versionRegex = regexp.MustCompile(`^(\d+)\.(\d+).(\d+).*$`)
+
+// VersionFromRelease converts a release string with format
+// 4.4.2[-1] to a kernel version number in LINUX_VERSION_CODE format.
+// That is, for kernel "a.b.c", the version number will be (a<<16 + b<<8 + c)
+func VersionFromRelease(releaseString string) (uint32, error) {
+	versionParts := versionRegex.FindStringSubmatch(releaseString)
+	if len(versionParts) != 4 {
+		return 0, fmt.Errorf("got invalid release version %q (expected format '4.3.2-1')", releaseString)
+	}
+	major, err := strconv.Atoi(versionParts[1])
+	if err != nil {
+		return 0, err
+	}
+
+	minor, err := strconv.Atoi(versionParts[2])
+	if err != nil {
+		return 0, err
+	}
+
+	patch, err := strconv.Atoi(versionParts[3])
+	if err != nil {
+		return 0, err
+	}
+	return Version(major, minor, patch), nil
+}
+
+// Version implements KERNEL_VERSION equivalent macro
+// #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
+func Version(major, minor, patch int) uint32 {
+	if patch > 255 {
+		patch = 255
+	}
+	out := major<<16 + minor<<8 + patch
+	return uint32(out)
+}
+
+func currentVersionUname() (uint32, error) {
+	var buf syscall.Utsname
+	if err := syscall.Uname(&buf); err != nil {
+		return 0, err
+	}
+	releaseString := strings.Trim(utsnameStr(buf.Release[:]), "\x00")
+	return VersionFromRelease(releaseString)
+}
+
+func currentVersionUbuntu() (uint32, error) {
+	procVersion, err := ioutil.ReadFile("/proc/version_signature")
+	if err != nil {
+		return 0, err
+	}
+	var u1, u2, releaseString string
+	_, err = fmt.Sscanf(string(procVersion), "%s %s %s", &u1, &u2, &releaseString)
+	if err != nil {
+		return 0, err
+	}
+	return VersionFromRelease(releaseString)
+}
+
+var debianVersionRegex = regexp.MustCompile(`.* SMP Debian (\d+\.\d+.\d+-\d+)(?:\+[[:alnum:]]*)?.*`)
+
+func parseDebianVersion(str string) (uint32, error) {
+	match := debianVersionRegex.FindStringSubmatch(str)
+	if len(match) != 2 {
+		return 0, fmt.Errorf("failed to parse kernel version from /proc/version: %s", str)
+	}
+	return VersionFromRelease(match[1])
+}
+
+func currentVersionDebian() (uint32, error) {
+	procVersion, err := ioutil.ReadFile("/proc/version")
+	if err != nil {
+		return 0, fmt.Errorf("error reading /proc/version: %s", err)
+	}
+
+	return parseDebianVersion(string(procVersion))
+}
+
+// CurrentVersion returns the current kernel version in
+// LINUX_VERSION_CODE format (see VersionFromRelease())
+func CurrentVersion() (uint32, error) {
+	// We need extra checks for Debian and Ubuntu as they modify
+	// the kernel version patch number for compatibility with
+	// out-of-tree modules. Linux perf tools do the same for Ubuntu
+	// systems: https://github.com/torvalds/linux/commit/d18acd15c
+	//
+	// See also:
+	// https://kernel-team.pages.debian.net/kernel-handbook/ch-versions.html
+	// https://wiki.ubuntu.com/Kernel/FAQ
+	version, err := currentVersionUbuntu()
+	if err == nil {
+		return version, nil
+	}
+	version, err = currentVersionDebian()
+	if err == nil {
+		return version, nil
+	}
+	return currentVersionUname()
+}

internal/kernel/kernel_other.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !linux
+// +build !linux
+
+package kernel
+
+// VersionFromRelease only implemented on Linux.
+func VersionFromRelease(_ string) (uint32, error) {
+	return 0, nil
+}
+
+// Version only implemented on Linux.
+func Version(_, _, _ int) uint32 {
+	return 0
+}
+
+// CurrentVersion only implemented on Linux.
+func CurrentVersion() (uint32, error) {
+	return 0, nil
+}

internal/kernel/kernel_test.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build linux
+// +build linux
+
+package kernel
+
+import "testing"
+
+var testData = []struct {
+	success       bool
+	releaseString string
+	kernelVersion uint32
+}{
+	{true, "4.1.2-3", 262402},
+	{true, "4.8.14-200.fc24.x86_64", 264206},
+	{true, "4.1.2-3foo", 262402},
+	{true, "4.1.2foo-1", 262402},
+	{true, "4.1.2-rkt-v1", 262402},
+	{true, "4.1.2rkt-v1", 262402},
+	{true, "4.1.2-3 foo", 262402},
+	{true, "3.10.0-1062.el7.x86_64", 199168},
+	{true, "3.0.0", 196608},
+	{true, "2.6.32", 132640},
+	{true, "5.13.0-30-generic", 331008},
+	{true, "5.10.0-1052-oem", 330240},
+	{false, "foo 4.1.2-3", 0},
+	{true, "4.1.2", 262402},
+	{false, ".4.1.2", 0},
+	{false, "4.1.", 0},
+	{false, "4.1", 0},
+}
+
+func TestVersionFromRelease(t *testing.T) {
+	for _, test := range testData {
+		version, err := VersionFromRelease(test.releaseString)
+		if err != nil && test.success {
+			t.Errorf("expected %q to success: %s", test.releaseString, err)
+		} else if err == nil && !test.success {
+			t.Errorf("expected %q to fail", test.releaseString)
+		}
+		if version != test.kernelVersion {
+			t.Errorf("expected kernel version %d, got %d", test.kernelVersion, version)
+		}
+	}
+}
+
+func TestParseDebianVersion(t *testing.T) {
+	for _, tc := range []struct {
+		success       bool
+		releaseString string
+		kernelVersion uint32
+	}{
+		// 4.9.168
+		{true, "Linux version 4.9.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.168-1+deb9u3 (2019-06-16)", 264616},
+		// 4.9.88
+		{true, "Linux ip-10-0-75-49 4.9.0-6-amd64 #1 SMP Debian 4.9.88-1+deb9u1 (2018-05-07) x86_64 GNU/Linux", 264536},
+		// 3.0.4
+		{true, "Linux version 3.16.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 4.9.2 (Debian 4.9.2-10+deb8u2) ) #1 SMP Debian 3.16.68-1 (2019-05-22)", 200772},
+		// Invalid
+		{false, "Linux version 4.9.125-linuxkit (root@659b6d51c354) (gcc version 6.4.0 (Alpine 6.4.0) ) #1 SMP Fri Sep 7 08:20:28 UTC 2018", 0},
+	} {
+		version, err := parseDebianVersion(tc.releaseString)
+		if err != nil && tc.success {
+			t.Errorf("expected %q to success: %s", tc.releaseString, err)
+		} else if err == nil && !tc.success {
+			t.Errorf("expected %q to fail", tc.releaseString)
+		}
+		if version != tc.kernelVersion {
+			t.Errorf("expected kernel version %d, got %d", tc.kernelVersion, version)
+		}
+	}
+}

internal/kernel/kernel_utsname_int8.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build (linux && 386) || (linux && amd64) || (linux && arm64) || (linux && mips64) || (linux && mips)
+// +build linux,386 linux,amd64 linux,arm64 linux,mips64 linux,mips
+
+package kernel
+
+func utsnameStr(in []int8) string {
+	out := make([]byte, 0, len(in))
+	for i := 0; i < len(in); i++ {
+		if in[i] == 0x00 {
+			break
+		}
+		out = append(out, byte(in[i]))
+	}
+	return string(out)
+}

internal/kernel/kernel_utsname_uint8.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build (linux && arm) || (linux && ppc64) || (linux && ppc64le) || (linux && s390x)
+// +build linux,arm linux,ppc64 linux,ppc64le linux,s390x
+
+package kernel
+
+func utsnameStr(in []uint8) string {
+	out := make([]byte, 0, len(in))
+	for i := 0; i < len(in); i++ {
+		if in[i] == 0x00 {
+			break
+		}
+		out = append(out, byte(in[i]))
+	}
+	return string(out)
+}