fix: attempt to drain the ReadFileStream for connection pooling (#12208)

avoid time_wait build up with getObject requests if there are
pending callers and they timeout, can lead to time_wait states

Bonus share the same buffer pool with erasure healing logic,
additionally also fixes a race where parallel readers were
never cleanup during Encode() phase, because pipe.Reader end
was never closed().

Added closer right away upon an error during Encode to make
sure to avoid racy Close() while stream was still being
Read().

cmd/erasure-heal_test.go

@@ -24,6 +24,9 @@ import (
 	"io"
 	"os"
 	"testing"
+
+	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/pkg/bpool"
 )
 
 var erasureHealTests = []struct {
@@ -136,8 +139,15 @@ func TestErasureHeal(t *testing.T) {
 				erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize(), true)
 		}
 
+		// Number of buffers, max 2GB
+		n := (2 * humanize.GiByte) / (int(test.blocksize) * 2)
+
+		// Initialize byte pool once for all sets, bpool size is set to
+		// setCount * setDriveCount with each memory upto blockSizeV2.
+		bp := bpool.NewBytePoolCap(n, int(test.blocksize), int(test.blocksize)*2)
+
 		// test case setup is complete - now call Heal()
-		err = erasure.Heal(context.Background(), readers, staleWriters, test.size)
+		err = erasure.Heal(context.Background(), readers, staleWriters, test.size, bp)
 		closeBitrotReaders(readers)
 		closeBitrotWriters(staleWriters)
 		if err != nil && !test.shouldFail {