MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-11 15:03:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

optimize readdir() open calls to be dealt with directly via 'fd' (#18762)

Browse Source
This commit is contained in:
Harshavardhana 2024-01-10 08:48:50 -08:00 committed by GitHub
parent e31081d79d
commit 39f9350697
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 43 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/os-instrumented.go
View File

@ -39,6 +39,8 @@ const (
osMetricRename
osMetricOpenFileW
osMetricOpenFileR
osMetricOpenFileWFd
osMetricOpenFileRFd
osMetricOpen
osMetricOpenFileDirectIO
osMetricLstat

33
cmd/os_unix.go
View File

@ -39,6 +39,22 @@ func access(name string) error {
return nil
}
// openFileWithFD return 'fd' based file descriptor
func openFileWithFD(name string, flag int, perm os.FileMode) (fd int, err error) {
switch flag & writeMode {
case writeMode:
defer updateOSMetrics(osMetricOpenFileWFd, name)(err)
default:
defer updateOSMetrics(osMetricOpenFileRFd, name)(err)
}
var e error
fd, e = syscall.Open(name, flag|syscall.O_CLOEXEC, uint32(perm))
if e != nil {
return -1, &os.PathError{Op: "open", Path: name, Err: e}
}
return fd, nil
}
// Forked from Golang but chooses to avoid performing lookup
//
// osMkdirAll creates a directory named path,
@ -149,7 +165,7 @@ func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err er
// the directory itself, if the dirPath doesn't exist this function doesn't return
// an error.
func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) error {
f, err := OpenFile(dirPath, readMode, 0o666)
fd, err := openFileWithFD(dirPath, readMode, 0o666)
if err != nil {
if osErrToFileErr(err) == errFileNotFound {
return nil
@ -160,15 +176,16 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
// There may be permission error when dirPath
// is at the root of the disk mount that may
// not have the permissions to avoid 'noatime'
f, err = Open(dirPath)
fd, err = openFileWithFD(dirPath, os.O_RDONLY, 0o666)
if err != nil {
if osErrToFileErr(err) == errFileNotFound {
return nil
}
return osErrToFileErr(err)
}
}
defer f.Close()
defer syscall.Close(fd)
bufp := direntPool.Get().(*[]byte)
defer direntPool.Put(bufp)
@ -181,7 +198,7 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
if boff >= nbuf {
boff = 0
stop := globalOSMetrics.time(osMetricReadDirent)
nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
nbuf, err = syscall.ReadDirent(fd, buf)
stop()
if err != nil {
if isSysErrNotDir(err) {
@ -241,7 +258,7 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
// Return count entries at the directory dirPath and all entries
// if count is set to -1
func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
f, err := OpenFile(dirPath, readMode, 0o666)
fd, err := openFileWithFD(dirPath, readMode, 0o666)
if err != nil {
if !osIsPermission(err) {
return nil, osErrToFileErr(err)
@ -249,12 +266,12 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
// There may be permission error when dirPath
// is at the root of the disk mount that may
// not have the permissions to avoid 'noatime'
f, err = Open(dirPath)
fd, err = openFileWithFD(dirPath, os.O_RDONLY, 0o666)
if err != nil {
return nil, osErrToFileErr(err)
}
}
defer f.Close()
defer syscall.Close(fd)
bufp := direntPool.Get().(*[]byte)
defer direntPool.Put(bufp)
@ -273,7 +290,7 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
if boff >= nbuf {
boff = 0
stop := globalOSMetrics.time(osMetricReadDirent)
nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
nbuf, err = syscall.ReadDirent(fd, buf)
stop()
if err != nil {
if isSysErrNotDir(err) {

30
cmd/osmetric_string.go
View File

@ -14,23 +14,25 @@ func _() {
_ = x[osMetricRename-3]
_ = x[osMetricOpenFileW-4]
_ = x[osMetricOpenFileR-5]
_ = x[osMetricOpen-6]
_ = x[osMetricOpenFileDirectIO-7]
_ = x[osMetricLstat-8]
_ = x[osMetricRemove-9]
_ = x[osMetricStat-10]
_ = x[osMetricAccess-11]
_ = x[osMetricCreate-12]
_ = x[osMetricReadDirent-13]
_ = x[osMetricFdatasync-14]
_ = x[osMetricSync-15]
_ = x[osMetricRename2-16]
_ = x[osMetricLast-17]
_ = x[osMetricOpenFileWFd-6]
_ = x[osMetricOpenFileRFd-7]
_ = x[osMetricOpen-8]
_ = x[osMetricOpenFileDirectIO-9]
_ = x[osMetricLstat-10]
_ = x[osMetricRemove-11]
_ = x[osMetricStat-12]
_ = x[osMetricAccess-13]
_ = x[osMetricCreate-14]
_ = x[osMetricReadDirent-15]
_ = x[osMetricFdatasync-16]
_ = x[osMetricSync-17]
_ = x[osMetricRename2-18]
_ = x[osMetricLast-19]
}
const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileWOpenFileROpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncRename2Last"
const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileWOpenFileROpenFileWFdOpenFileRFdOpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncRename2Last"
var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 37, 46, 50, 66, 71, 77, 81, 87, 93, 103, 112, 116, 123, 127}
var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 37, 46, 57, 68, 72, 88, 93, 99, 103, 109, 115, 125, 134, 138, 145, 149}
func (i osMetric) String() string {
if i >= osMetric(len(_osMetric_index)-1) {