From e20e08d700c5d6ef1811a07e056c872ec750f099 Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Wed, 8 Apr 2020 19:42:57 -0700
Subject: [PATCH] fix: remove the sleep from listing operations (#9287)

make rest of the Walk() function more predictable,
it was observed that in nominal deployments even
without much workload the drives are generally
slow for respond for readdir operations, for the
sleepDuration factor of 10 this can cause
unexpected slowness in the Listing calls, while
it is good for all other I/O, it may simply slow
down Listing immensely which is not useful.

fixes #9261
---
 cmd/posix.go | 45 ++++++++++++++++++++-------------------------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/cmd/posix.go b/cmd/posix.go
index b6c09771e..7ec9fe77e 100644
--- a/cmd/posix.go
+++ b/cmd/posix.go
@@ -351,7 +351,7 @@ func (s *posix) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache)
 			return 0, errSkipFile
 		}
 
-		meta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaBuf)
+		meta, err := xlMetaV1UnmarshalJSON(ctx, xlMetaBuf)
 		if err != nil {
 			return 0, nil
 		}
@@ -677,9 +677,8 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
 	if err != nil {
 		return nil, err
 	}
-	// Stat a volume entry.
-	_, err = os.Stat((volumeDir))
-	if err != nil {
+
+	if _, err = os.Stat(volumeDir); err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
 		} else if isSysErrIO(err) {
@@ -694,6 +693,9 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
 	} else {
 		entries, err = readDir(dirPath)
 	}
+	if err != nil {
+		return nil, err
+	}
 
 	for i, entry := range entries {
 		if entry != receiptJSON {
@@ -704,7 +706,7 @@ func (s *posix) ListDirSplunk(volume, dirPath string, count int) (entries []stri
 		}
 	}
 
-	return entries, err
+	return entries, nil
 }
 
 // WalkSplunk - is a sorted walker which returns file entries in lexically
@@ -732,10 +734,10 @@ func (s *posix) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan stru
 	ch = make(chan FileInfo)
 	go func() {
 		defer close(ch)
-		listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) {
+		listDir := func(volume, dirPath, dirEntry string) (bool, []string) {
 			entries, err := s.ListDirSplunk(volume, dirPath, -1)
 			if err != nil {
-				return
+				return false, nil
 			}
 			if len(entries) == 0 {
 				return true, nil
@@ -758,14 +760,11 @@ func (s *posix) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan stru
 					Mode:   os.ModeDir,
 				}
 			} else {
-				// Dynamic time delay.
-				t := UTCNow()
-				buf, err := s.ReadAll(volume, pathJoin(walkResult.entry, xlMetaJSONFile))
-				sleepDuration(time.Since(t), 10.0)
+				xlMetaBuf, err := ioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, xlMetaJSONFile))
 				if err != nil {
 					continue
 				}
-				fi = readMetadata(buf, volume, walkResult.entry)
+				fi = readMetadata(xlMetaBuf, volume, walkResult.entry)
 			}
 			select {
 			case ch <- fi:
@@ -809,13 +808,10 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
 	ch = make(chan FileInfo, maxObjectList)
 	go func() {
 		defer close(ch)
-		listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) {
-			// Dynamic time delay.
-			t := UTCNow()
+		listDir := func(volume, dirPath, dirEntry string) (bool, []string) {
 			entries, err := s.ListDir(volume, dirPath, -1, leafFile)
-			sleepDuration(time.Since(t), 10.0)
 			if err != nil {
-				return
+				return false, nil
 			}
 			if len(entries) == 0 {
 				return true, nil
@@ -838,14 +834,11 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
 					Mode:   os.ModeDir,
 				}
 			} else {
-				// Dynamic time delay.
-				t := UTCNow()
-				buf, err := s.ReadAll(volume, pathJoin(walkResult.entry, leafFile))
-				sleepDuration(time.Since(t), 10.0)
+				xlMetaBuf, err := ioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, leafFile))
 				if err != nil {
 					continue
 				}
-				fi = readMetadataFn(buf, volume, walkResult.entry)
+				fi = readMetadataFn(xlMetaBuf, volume, walkResult.entry)
 			}
 			select {
 			case ch <- fi:
@@ -871,9 +864,8 @@ func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (ent
 	if err != nil {
 		return nil, err
 	}
-	// Stat a volume entry.
-	_, err = os.Stat((volumeDir))
-	if err != nil {
+
+	if _, err = os.Stat(volumeDir); err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
 		} else if isSysErrIO(err) {
@@ -888,6 +880,9 @@ func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (ent
 	} else {
 		entries, err = readDir(dirPath)
 	}
+	if err != nil {
+		return nil, err
+	}
 
 	// If leaf file is specified, filter out the entries.
 	if leafFile != "" {