From 0680af7414f8ea06bcb523653356058620a75e29 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Tue, 23 Jul 2024 09:36:24 -0700 Subject: [PATCH] Set O_NONBLOCK for reads and writes on unix (#20133) Tracing syscalls, opening and reading an `xl.meta` looks like this: ``` openat(AT_FDCWD, "/mnt/drive1/ss8-old/testbucket/ObjSize4MiBThreads72/(554O51H/peTb(0iztdbTKw59.csv/xl.meta", O_RDONLY|O_NOATIME|O_CLOEXEC) = 34 <0.000> fcntl(34, F_GETFL) = 0x48000 (flags O_RDONLY|O_LARGEFILE|O_NOATIME) <0.000> fcntl(34, F_SETFL, O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_NOATIME) = 0 <0.000> epoll_ctl(4, EPOLL_CTL_ADD, 34, {events=EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, data={u32=3172471557, u64=8145488475984499461}}) = -1 EPERM (Operation not permitted) <0.000> fcntl(34, F_GETFL) = 0x48800 (flags O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_NOATIME) <0.000> fcntl(34, F_SETFL, O_RDONLY|O_LARGEFILE|O_NOATIME) = 0 <0.000> fstat(34, {st_mode=S_IFREG|0644, st_size=354, ...}) = 0 <0.000> read(34, "XL2 \1\0\3\0\306\0\0\1P\2\2\1\304$\225\304\20\0\0\0\0\0\0\0\0\0\0\0"..., 354) = 354 <0.000> close(34) = 0 <0.000> ``` Everything until `fstat` is the `os.Open` call. Looking at the code: https://github.com/golang/go/blob/master/src/os/file_unix.go#L212-L243 It seems for every file it "tries" to see if it is pollable. This causes `syscall.SetNonblock(fd, true)` to be called. This is the first `F_SETFL`. It then calls `f.pfd.Init("file", true)`. This will attempt to set it as pollable using `epoll_ctl`. This will always fail for files. It therefore calls `syscall.SetNonblock(fd, false)` resulting in the second `F_SETFL`. If we set the `O_NONBLOCK` call on the initial open, we should avoid the 4 `fcntl` syscalls per file. I don't see any way to avoid the `epoll_ctl` call, since kind is either `kindOpenFile` or `kindNonBlock`, so "pollable" will always be true. However avoiding 4 of 6 syscalls still seems worth it. This should not have any effect, since files will end up with "nonblock" anyway. --- cmd/xl-storage_noatime_notsupported.go | 3 +-- cmd/xl-storage_noatime_supported.go | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/xl-storage_noatime_notsupported.go b/cmd/xl-storage_noatime_notsupported.go index ac6718ef6..c9c91d624 100644 --- a/cmd/xl-storage_noatime_notsupported.go +++ b/cmd/xl-storage_noatime_notsupported.go @@ -1,5 +1,4 @@ -//go:build windows || darwin || freebsd -// +build windows darwin freebsd +//go:build !unix || darwin || freebsd // Copyright (c) 2015-2021 MinIO, Inc. // diff --git a/cmd/xl-storage_noatime_supported.go b/cmd/xl-storage_noatime_supported.go index eefeef87c..efa75ff0a 100644 --- a/cmd/xl-storage_noatime_supported.go +++ b/cmd/xl-storage_noatime_supported.go @@ -1,5 +1,4 @@ -//go:build !windows && !darwin && !freebsd -// +build !windows,!darwin,!freebsd +//go:build unix && !darwin && !freebsd // Copyright (c) 2015-2021 MinIO, Inc. // @@ -22,12 +21,14 @@ package cmd import ( "os" + "syscall" ) var ( // Disallow updating access times - readMode = os.O_RDONLY | 0x40000 // O_NOATIME + // Add non-block to avoid syscall to attempt to set epoll on files. + readMode = os.O_RDONLY | 0x40000 | syscall.O_NONBLOCK // O_NOATIME // Write with data sync only used only for `xl.meta` writes - writeMode = 0x1000 // O_DSYNC + writeMode = 0x1000 | syscall.O_NONBLOCK // O_DSYNC )