MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

de-couple walkMu and walkReadMu for some granularity (#13231)

Browse Source 
This commit brings two locks instead of single lock for
WalkDir() calls on top of c25816eabc.

The main reason is to avoid contention between readMetadata()
and ListDir() calls, ListDir() can take time on prefixes that
are huge for readdir() but this shouldn't end up blocking
all readMetadata() operations, this allows for more room for
I/O while not overly penalizing all listing operations.
This commit is contained in:
Harshavardhana 2021-09-17 12:14:12 -07:00 committed by GitHub
parent 1fc0e9a6aa
commit 66fcd02aa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/metacache-walk.go
View File

@ -121,6 +121,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
if contextCanceled(ctx) {
return ctx.Err()
}
s.walkMu.Lock()
entries, err := s.ListDir(ctx, opts.Bucket, current, -1)
s.walkMu.Unlock()
@ -173,9 +174,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
// If root was an object return it as such.
if HasSuffix(entry, xlStorageFormatFile) {
var meta metaCacheEntry
s.walkMu.Lock()
s.walkReadMu.Lock()
meta.metadata, err = s.readMetadata(pathJoin(volumeDir, current, entry))
s.walkMu.Unlock()
s.walkReadMu.Unlock()
if err != nil {
logger.LogIf(ctx, err)
continue
@ -190,9 +191,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
// Check legacy.
if HasSuffix(entry, xlStorageFormatFileV1) {
var meta metaCacheEntry
s.walkMu.Lock()
s.walkReadMu.Lock()
meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, current, entry))
s.walkMu.Unlock()
s.walkReadMu.Unlock()
if err != nil {
logger.LogIf(ctx, err)
continue
@ -248,9 +249,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash
}
s.walkMu.Lock()
s.walkReadMu.Lock()
meta.metadata, err = s.readMetadata(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
s.walkMu.Unlock()
s.walkReadMu.Unlock()
switch {
case err == nil:
// It was an object
@ -259,9 +260,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
}
out <- meta
case osIsNotExist(err), isSysErrIsDir(err):
s.walkMu.Lock()
meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1))
s.walkMu.Unlock()
if err == nil {
// It was an object
out <- meta

3
cmd/xl-storage.go
View File

@ -142,7 +142,8 @@ type xlStorage struct {
sync.RWMutex
// mutex to prevent concurrent read operations overloading walks.
walkMu sync.Mutex
walkMu sync.Mutex
walkReadMu sync.Mutex
}
// checkPathLength - returns error if given path name length more than 255