MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-26 07:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove the sleep from listing operations (#9287)

Browse Source 
make rest of the Walk() function more predictable,
it was observed that in nominal deployments even
without much workload the drives are generally
slow for respond for readdir operations, for the
sleepDuration factor of 10 this can cause
unexpected slowness in the Listing calls, while
it is good for all other I/O, it may simply slow
down Listing immensely which is not useful.

fixes #9261
This commit is contained in:
Harshavardhana 2020-04-08 19:42:57 -07:00 committed by GitHub
parent ac07df2985
commit e20e08d700
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 20 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
cmd/posix.go
View File

@ -351,7 +351,7 @@ func (s *posix) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache)
return 0, errSkipFile return 0, errSkipFile
} }
meta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaBuf) meta, err := xlMetaV1UnmarshalJSON(ctx, xlMetaBuf)
if err != nil { if err != nil {
return 0, nil return 0, nil
} }
@ -677,9 +677,8 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Stat a volume entry.
_, err = os.Stat((volumeDir)) if _, err = os.Stat(volumeDir); err != nil {
if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil, errVolumeNotFound return nil, errVolumeNotFound
} else if isSysErrIO(err) { } else if isSysErrIO(err) {
@ -694,6 +693,9 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
} else { } else {
entries, err = readDir(dirPath) entries, err = readDir(dirPath)
} }
if err != nil {
return nil, err
}
for i, entry := range entries { for i, entry := range entries {
if entry != receiptJSON { if entry != receiptJSON {
@ -704,7 +706,7 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
} }
} }
return entries, err return entries, nil
} }
// WalkSplunk - is a sorted walker which returns file entries in lexically // WalkSplunk - is a sorted walker which returns file entries in lexically
@ -732,10 +734,10 @@ func (s *posix) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan stru
ch = make(chan FileInfo) ch = make(chan FileInfo)
go func() { go func() {
defer close(ch) defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) { listDir := func(volume, dirPath, dirEntry string) (bool, []string) {
entries, err := s.ListDirSplunk(volume, dirPath, -1) entries, err := s.ListDirSplunk(volume, dirPath, -1)
if err != nil { if err != nil {
return return false, nil
} }
if len(entries) == 0 { if len(entries) == 0 {
return true, nil return true, nil
@ -758,14 +760,11 @@ func (s *posix) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan stru
Mode: os.ModeDir, Mode: os.ModeDir,
} }
} else { } else {
// Dynamic time delay. xlMetaBuf, err := ioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, xlMetaJSONFile))
t := UTCNow()
buf, err := s.ReadAll(volume, pathJoin(walkResult.entry, xlMetaJSONFile))
sleepDuration(time.Since(t), 10.0)
if err != nil { if err != nil {
continue continue
} }
fi = readMetadata(buf, volume, walkResult.entry) fi = readMetadata(xlMetaBuf, volume, walkResult.entry)
} }
select { select {
case ch <- fi: case ch <- fi:
@ -809,13 +808,10 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
ch = make(chan FileInfo, maxObjectList) ch = make(chan FileInfo, maxObjectList)
go func() { go func() {
defer close(ch) defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) { listDir := func(volume, dirPath, dirEntry string) (bool, []string) {
// Dynamic time delay.
t := UTCNow()
entries, err := s.ListDir(volume, dirPath, -1, leafFile) entries, err := s.ListDir(volume, dirPath, -1, leafFile)
sleepDuration(time.Since(t), 10.0)
if err != nil { if err != nil {
return return false, nil
} }
if len(entries) == 0 { if len(entries) == 0 {
return true, nil return true, nil
@ -838,14 +834,11 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
Mode: os.ModeDir, Mode: os.ModeDir,
} }
} else { } else {
// Dynamic time delay. xlMetaBuf, err := ioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, leafFile))
t := UTCNow()
buf, err := s.ReadAll(volume, pathJoin(walkResult.entry, leafFile))
sleepDuration(time.Since(t), 10.0)
if err != nil { if err != nil {
continue continue
} }
fi = readMetadataFn(buf, volume, walkResult.entry) fi = readMetadataFn(xlMetaBuf, volume, walkResult.entry)
} }
select { select {
case ch <- fi: case ch <- fi:
@ -871,9 +864,8 @@ func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (ent
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Stat a volume entry.
_, err = os.Stat((volumeDir)) if _, err = os.Stat(volumeDir); err != nil {
if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil, errVolumeNotFound return nil, errVolumeNotFound
} else if isSysErrIO(err) { } else if isSysErrIO(err) {
@ -888,6 +880,9 @@ func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (ent
} else { } else {
entries, err = readDir(dirPath) entries, err = readDir(dirPath)
} }
if err != nil {
return nil, err
}
// If leaf file is specified, filter out the entries. // If leaf file is specified, filter out the entries.
if leafFile != "" { if leafFile != "" {