From 91d419ee6c8bdbf4a66e6727b02e16bcad316c4d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 10 Mar 2022 17:36:13 -0800 Subject: [PATCH] warn issues about large block I/O performance for Linux older than 4.0.0 (#14524) This PR simply adds a warning message when it detects older kernel versions and warn's them about potential performance issues on this kernel. The issue can be seen only with parallel I/O across all drives on denser setups such as 90 drives or 45 drives per server configurations. --- cmd/server-main.go | 5 + cmd/server-rlimit.go | 18 ++++ cmd/xl-storage.go | 9 +- internal/kernel/kernel.go | 130 ++++++++++++++++++++++++ internal/kernel/kernel_other.go | 36 +++++++ internal/kernel/kernel_test.go | 88 ++++++++++++++++ internal/kernel/kernel_utsname_int8.go | 32 ++++++ internal/kernel/kernel_utsname_uint8.go | 32 ++++++ 8 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 internal/kernel/kernel.go create mode 100644 internal/kernel/kernel_other.go create mode 100644 internal/kernel/kernel_test.go create mode 100644 internal/kernel/kernel_utsname_int8.go create mode 100644 internal/kernel/kernel_utsname_uint8.go diff --git a/cmd/server-main.go b/cmd/server-main.go index b550e8bad..c299aae22 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -448,6 +448,11 @@ func serverMain(ctx *cli.Context) { // Set system resources to maximum. setMaxResources() + // Verify kernel release and version. + if oldLinux() { + logger.Info(color.RedBold("WARNING: Detected Linux kernel version older than 4.0.0 release, there are some known potential performance problems with this kernel version. MinIO recommends a minimum of 4.x.x linux kernel version for best performance")) + } + // Configure server. handler, err := configureServerHandler(globalEndpoints) if err != nil { diff --git a/cmd/server-rlimit.go b/cmd/server-rlimit.go index ec3acb3cd..52d4b0d73 100644 --- a/cmd/server-rlimit.go +++ b/cmd/server-rlimit.go @@ -21,10 +21,28 @@ import ( "runtime" "runtime/debug" + "github.com/minio/minio/internal/kernel" "github.com/minio/minio/internal/logger" "github.com/minio/pkg/sys" ) +func oldLinux() bool { + currentKernel, err := kernel.CurrentVersion() + if err != nil { + // Could not probe the kernel version + return false + } + + if currentKernel == 0 { + // We could not get any valid value return false + return false + } + + // legacy linux indicator for printing warnings + // about older Linux kernels and Go runtime. + return currentKernel < kernel.Version(4, 0, 0) +} + func setMaxResources() (err error) { // Set the Go runtime max threads threshold to 90% of kernel setting. sysMaxThreads, mErr := sys.GetMaxThreads() diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 800d4bbc0..d6149acc0 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -49,11 +49,12 @@ import ( const ( nullVersionID = "null" - // Really large streams threshold per shard. - reallyLargeFileThreshold = 64 * humanize.MiByte // Optimized for HDDs + // Largest streams threshold per shard. + largestFileThreshold = 64 * humanize.MiByte // Optimized for HDDs // Small file threshold below which data accompanies metadata from storage layer. smallFileThreshold = 128 * humanize.KiByte // Optimized for NVMe/SSDs + // For hardrives it is possible to set this to a lower value to avoid any // spike in latency. But currently we are simply keeping it optimal for SSDs. @@ -1786,8 +1787,8 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz }() var bufp *[]byte - if fileSize > 0 && fileSize >= reallyLargeFileThreshold { - // use a larger 4MiB buffer for really large streams. + if fileSize > 0 && fileSize >= largestFileThreshold { + // use a larger 4MiB buffer for a really large streams. bufp = xioutil.ODirectPoolXLarge.Get().(*[]byte) defer xioutil.ODirectPoolXLarge.Put(bufp) } else { diff --git a/internal/kernel/kernel.go b/internal/kernel/kernel.go new file mode 100644 index 000000000..d74419602 --- /dev/null +++ b/internal/kernel/kernel.go @@ -0,0 +1,130 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build linux +// +build linux + +package kernel + +import ( + "fmt" + "io/ioutil" + "regexp" + "strconv" + "strings" + "syscall" +) + +var versionRegex = regexp.MustCompile(`^(\d+)\.(\d+).(\d+).*$`) + +// VersionFromRelease converts a release string with format +// 4.4.2[-1] to a kernel version number in LINUX_VERSION_CODE format. +// That is, for kernel "a.b.c", the version number will be (a<<16 + b<<8 + c) +func VersionFromRelease(releaseString string) (uint32, error) { + versionParts := versionRegex.FindStringSubmatch(releaseString) + if len(versionParts) != 4 { + return 0, fmt.Errorf("got invalid release version %q (expected format '4.3.2-1')", releaseString) + } + major, err := strconv.Atoi(versionParts[1]) + if err != nil { + return 0, err + } + + minor, err := strconv.Atoi(versionParts[2]) + if err != nil { + return 0, err + } + + patch, err := strconv.Atoi(versionParts[3]) + if err != nil { + return 0, err + } + return Version(major, minor, patch), nil +} + +// Version implements KERNEL_VERSION equivalent macro +// #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c))) +func Version(major, minor, patch int) uint32 { + if patch > 255 { + patch = 255 + } + out := major<<16 + minor<<8 + patch + return uint32(out) +} + +func currentVersionUname() (uint32, error) { + var buf syscall.Utsname + if err := syscall.Uname(&buf); err != nil { + return 0, err + } + releaseString := strings.Trim(utsnameStr(buf.Release[:]), "\x00") + return VersionFromRelease(releaseString) +} + +func currentVersionUbuntu() (uint32, error) { + procVersion, err := ioutil.ReadFile("/proc/version_signature") + if err != nil { + return 0, err + } + var u1, u2, releaseString string + _, err = fmt.Sscanf(string(procVersion), "%s %s %s", &u1, &u2, &releaseString) + if err != nil { + return 0, err + } + return VersionFromRelease(releaseString) +} + +var debianVersionRegex = regexp.MustCompile(`.* SMP Debian (\d+\.\d+.\d+-\d+)(?:\+[[:alnum:]]*)?.*`) + +func parseDebianVersion(str string) (uint32, error) { + match := debianVersionRegex.FindStringSubmatch(str) + if len(match) != 2 { + return 0, fmt.Errorf("failed to parse kernel version from /proc/version: %s", str) + } + return VersionFromRelease(match[1]) +} + +func currentVersionDebian() (uint32, error) { + procVersion, err := ioutil.ReadFile("/proc/version") + if err != nil { + return 0, fmt.Errorf("error reading /proc/version: %s", err) + } + + return parseDebianVersion(string(procVersion)) +} + +// CurrentVersion returns the current kernel version in +// LINUX_VERSION_CODE format (see VersionFromRelease()) +func CurrentVersion() (uint32, error) { + // We need extra checks for Debian and Ubuntu as they modify + // the kernel version patch number for compatibility with + // out-of-tree modules. Linux perf tools do the same for Ubuntu + // systems: https://github.com/torvalds/linux/commit/d18acd15c + // + // See also: + // https://kernel-team.pages.debian.net/kernel-handbook/ch-versions.html + // https://wiki.ubuntu.com/Kernel/FAQ + version, err := currentVersionUbuntu() + if err == nil { + return version, nil + } + version, err = currentVersionDebian() + if err == nil { + return version, nil + } + return currentVersionUname() +} diff --git a/internal/kernel/kernel_other.go b/internal/kernel/kernel_other.go new file mode 100644 index 000000000..998cff7f7 --- /dev/null +++ b/internal/kernel/kernel_other.go @@ -0,0 +1,36 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build !linux +// +build !linux + +package kernel + +// VersionFromRelease only implemented on Linux. +func VersionFromRelease(_ string) (uint32, error) { + return 0, nil +} + +// Version only implemented on Linux. +func Version(_, _, _ int) uint32 { + return 0 +} + +// CurrentVersion only implemented on Linux. +func CurrentVersion() (uint32, error) { + return 0, nil +} diff --git a/internal/kernel/kernel_test.go b/internal/kernel/kernel_test.go new file mode 100644 index 000000000..fa60b598c --- /dev/null +++ b/internal/kernel/kernel_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build linux +// +build linux + +package kernel + +import "testing" + +var testData = []struct { + success bool + releaseString string + kernelVersion uint32 +}{ + {true, "4.1.2-3", 262402}, + {true, "4.8.14-200.fc24.x86_64", 264206}, + {true, "4.1.2-3foo", 262402}, + {true, "4.1.2foo-1", 262402}, + {true, "4.1.2-rkt-v1", 262402}, + {true, "4.1.2rkt-v1", 262402}, + {true, "4.1.2-3 foo", 262402}, + {true, "3.10.0-1062.el7.x86_64", 199168}, + {true, "3.0.0", 196608}, + {true, "2.6.32", 132640}, + {true, "5.13.0-30-generic", 331008}, + {true, "5.10.0-1052-oem", 330240}, + {false, "foo 4.1.2-3", 0}, + {true, "4.1.2", 262402}, + {false, ".4.1.2", 0}, + {false, "4.1.", 0}, + {false, "4.1", 0}, +} + +func TestVersionFromRelease(t *testing.T) { + for _, test := range testData { + version, err := VersionFromRelease(test.releaseString) + if err != nil && test.success { + t.Errorf("expected %q to success: %s", test.releaseString, err) + } else if err == nil && !test.success { + t.Errorf("expected %q to fail", test.releaseString) + } + if version != test.kernelVersion { + t.Errorf("expected kernel version %d, got %d", test.kernelVersion, version) + } + } +} + +func TestParseDebianVersion(t *testing.T) { + for _, tc := range []struct { + success bool + releaseString string + kernelVersion uint32 + }{ + // 4.9.168 + {true, "Linux version 4.9.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.168-1+deb9u3 (2019-06-16)", 264616}, + // 4.9.88 + {true, "Linux ip-10-0-75-49 4.9.0-6-amd64 #1 SMP Debian 4.9.88-1+deb9u1 (2018-05-07) x86_64 GNU/Linux", 264536}, + // 3.0.4 + {true, "Linux version 3.16.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 4.9.2 (Debian 4.9.2-10+deb8u2) ) #1 SMP Debian 3.16.68-1 (2019-05-22)", 200772}, + // Invalid + {false, "Linux version 4.9.125-linuxkit (root@659b6d51c354) (gcc version 6.4.0 (Alpine 6.4.0) ) #1 SMP Fri Sep 7 08:20:28 UTC 2018", 0}, + } { + version, err := parseDebianVersion(tc.releaseString) + if err != nil && tc.success { + t.Errorf("expected %q to success: %s", tc.releaseString, err) + } else if err == nil && !tc.success { + t.Errorf("expected %q to fail", tc.releaseString) + } + if version != tc.kernelVersion { + t.Errorf("expected kernel version %d, got %d", tc.kernelVersion, version) + } + } +} diff --git a/internal/kernel/kernel_utsname_int8.go b/internal/kernel/kernel_utsname_int8.go new file mode 100644 index 000000000..1c7855117 --- /dev/null +++ b/internal/kernel/kernel_utsname_int8.go @@ -0,0 +1,32 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build (linux && 386) || (linux && amd64) || (linux && arm64) || (linux && mips64) || (linux && mips) +// +build linux,386 linux,amd64 linux,arm64 linux,mips64 linux,mips + +package kernel + +func utsnameStr(in []int8) string { + out := make([]byte, 0, len(in)) + for i := 0; i < len(in); i++ { + if in[i] == 0x00 { + break + } + out = append(out, byte(in[i])) + } + return string(out) +} diff --git a/internal/kernel/kernel_utsname_uint8.go b/internal/kernel/kernel_utsname_uint8.go new file mode 100644 index 000000000..848e7769b --- /dev/null +++ b/internal/kernel/kernel_utsname_uint8.go @@ -0,0 +1,32 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build (linux && arm) || (linux && ppc64) || (linux && ppc64le) || (linux && s390x) +// +build linux,arm linux,ppc64 linux,ppc64le linux,s390x + +package kernel + +func utsnameStr(in []uint8) string { + out := make([]byte, 0, len(in)) + for i := 0; i < len(in); i++ { + if in[i] == 0x00 { + break + } + out = append(out, byte(in[i])) + } + return string(out) +}