From 9efcb9e15ccb40421a11cef977de3f33a0587275 Mon Sep 17 00:00:00 2001
From: Klaus Post <klauspost@gmail.com>
Date: Fri, 26 Mar 2021 19:18:30 +0100
Subject: [PATCH] Fix listPathRaw/WalkDir cancelation (#11905)

In #11888 we observe a lot of running, WalkDir calls.

There doesn't appear to be any listerners for these calls, so they should be aborted.

Ensure that WalkDir aborts when upstream cancels the request.

Fixes #11888
---
 cmd/common-main.go    | 10 ++++++++++
 cmd/metacache-set.go  | 10 +++++++++-
 cmd/metacache-walk.go |  9 +++++++++
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/cmd/common-main.go b/cmd/common-main.go
index 79319ad8f..0550f6a5e 100644
--- a/cmd/common-main.go
+++ b/cmd/common-main.go
@@ -428,3 +428,13 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 	secureConn = true
 	return x509Certs, manager, secureConn, nil
 }
+
+// contextCanceled returns whether a context is canceled.
+func contextCanceled(ctx context.Context) bool {
+	select {
+	case <-ctx.Done():
+		return true
+	default:
+		return false
+	}
+}
diff --git a/cmd/metacache-set.go b/cmd/metacache-set.go
index 8be141da5..c1ab114e3 100644
--- a/cmd/metacache-set.go
+++ b/cmd/metacache-set.go
@@ -811,6 +811,9 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 	if len(disks) == 0 {
 		return fmt.Errorf("listPathRaw: 0 drives provided")
 	}
+	// Cancel upstream if we finish before we expect.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 
 	askDisks := len(disks)
 	readers := make([]*metacacheReader, askDisks)
@@ -821,6 +824,8 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 		if err != nil {
 			return err
 		}
+		// Make sure we close the pipe so blocked writes doesn't stay around.
+		defer r.CloseWithError(context.Canceled)
 		// Send request to each disk.
 		go func() {
 			werr := d.WalkDir(ctx, WalkDirOptions{
@@ -832,7 +837,10 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 				ForwardTo:      opts.forwardTo,
 			}, w)
 			w.CloseWithError(werr)
-			if werr != io.EOF && werr != nil && werr.Error() != errFileNotFound.Error() && werr.Error() != errVolumeNotFound.Error() {
+			if werr != io.EOF && werr != nil &&
+				werr.Error() != errFileNotFound.Error() &&
+				werr.Error() != errVolumeNotFound.Error() &&
+				!errors.Is(werr, context.Canceled) {
 				logger.LogIf(ctx, werr)
 			}
 		}()
diff --git a/cmd/metacache-walk.go b/cmd/metacache-walk.go
index 6b0b06da8..9e66b7065 100644
--- a/cmd/metacache-walk.go
+++ b/cmd/metacache-walk.go
@@ -107,6 +107,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	forward := opts.ForwardTo
 	var scanDir func(path string) error
 	scanDir = func(current string) error {
+		if contextCanceled(ctx) {
+			return ctx.Err()
+		}
 		entries, err := s.ListDir(ctx, opts.Bucket, current, -1)
 		if err != nil {
 			// Folder could have gone away in-between
@@ -142,6 +145,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			// Do do not retain the file.
 			entries[i] = ""
 
+			if contextCanceled(ctx) {
+				return ctx.Err()
+			}
 			// If root was an object return it as such.
 			if HasSuffix(entry, xlStorageFormatFile) {
 				var meta metaCacheEntry
@@ -183,6 +189,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			if entry == "" {
 				continue
 			}
+			if contextCanceled(ctx) {
+				return ctx.Err()
+			}
 			meta := metaCacheEntry{name: PathJoin(current, entry)}
 
 			// If directory entry on stack before this, pop it now.