MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-25 06:35:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove the sleep from listing operations (#9287)

Browse Source 
make rest of the Walk() function more predictable,
it was observed that in nominal deployments even
without much workload the drives are generally
slow for respond for readdir operations, for the
sleepDuration factor of 10 this can cause
unexpected slowness in the Listing calls, while
it is good for all other I/O, it may simply slow
down Listing immensely which is not useful.

fixes #9261
This commit is contained in:
Harshavardhana 2020-04-08 19:42:57 -07:00 committed by GitHub
parent ac07df2985
commit e20e08d700
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 20 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
cmd/posix.go
View File

@ -351,7 +351,7 @@ func (s *posix) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache)
return 0, errSkipFile
}
meta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaBuf)
meta, err := xlMetaV1UnmarshalJSON(ctx, xlMetaBuf)
if err != nil {
return 0, nil
}
@ -677,9 +677,8 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
if err != nil {
return nil, err
}
// Stat a volume entry.
_, err = os.Stat((volumeDir))
if err != nil {
if _, err = os.Stat(volumeDir); err != nil {
if os.IsNotExist(err) {
return nil, errVolumeNotFound
} else if isSysErrIO(err) {
@ -694,6 +693,9 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
} else {
entries, err = readDir(dirPath)
}
if err != nil {
return nil, err
}
for i, entry := range entries {
if entry != receiptJSON {
@ -704,7 +706,7 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
}
}
return entries, err
return entries, nil
}
// WalkSplunk - is a sorted walker which returns file entries in lexically
@ -732,10 +734,10 @@ func (s *posix) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan stru
ch = make(chan FileInfo)
go func() {
defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) {
listDir := func(volume, dirPath, dirEntry string) (bool, []string) {
entries, err := s.ListDirSplunk(volume, dirPath, -1)
if err != nil {
return
return false, nil
}
if len(entries) == 0 {
return true, nil
@ -758,14 +760,11 @@ func (s *posix) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan stru
Mode: os.ModeDir,
}
} else {
// Dynamic time delay.
t := UTCNow()
buf, err := s.ReadAll(volume, pathJoin(walkResult.entry, xlMetaJSONFile))
sleepDuration(time.Since(t), 10.0)
xlMetaBuf, err := ioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, xlMetaJSONFile))
if err != nil {
continue
}
fi = readMetadata(buf, volume, walkResult.entry)
fi = readMetadata(xlMetaBuf, volume, walkResult.entry)
}
select {
case ch <- fi:
@ -809,13 +808,10 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
ch = make(chan FileInfo, maxObjectList)
go func() {
defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) {
// Dynamic time delay.
t := UTCNow()
listDir := func(volume, dirPath, dirEntry string) (bool, []string) {
entries, err := s.ListDir(volume, dirPath, -1, leafFile)
sleepDuration(time.Since(t), 10.0)
if err != nil {
return
return false, nil
}
if len(entries) == 0 {
return true, nil
@ -838,14 +834,11 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
Mode: os.ModeDir,
}
} else {
// Dynamic time delay.
t := UTCNow()
buf, err := s.ReadAll(volume, pathJoin(walkResult.entry, leafFile))
sleepDuration(time.Since(t), 10.0)
xlMetaBuf, err := ioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, leafFile))
if err != nil {
continue
}
fi = readMetadataFn(buf, volume, walkResult.entry)
fi = readMetadataFn(xlMetaBuf, volume, walkResult.entry)
}
select {
case ch <- fi:
@ -871,9 +864,8 @@ func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (ent
if err != nil {
return nil, err
}
// Stat a volume entry.
_, err = os.Stat((volumeDir))
if err != nil {
if _, err = os.Stat(volumeDir); err != nil {
if os.IsNotExist(err) {
return nil, errVolumeNotFound
} else if isSysErrIO(err) {
@ -888,6 +880,9 @@ func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (ent
} else {
entries, err = readDir(dirPath)
}
if err != nil {
return nil, err
}
// If leaf file is specified, filter out the entries.
if leafFile != "" {