[feat] Add configurable deadline for writers (#11822)

This PR adds deadlines per Write() calls, such
that slow drives are timed-out appropriately and
the overall responsiveness for Writes() is always
up to a predefined threshold providing applications
sustained latency even if one of the drives is slow
to respond.

cmd/bitrot-streaming.go

@@ -23,8 +23,11 @@ import (
 	"fmt"
 	"hash"
 	"io"
+	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
+	xioutil "github.com/minio/minio/pkg/ioutil"
 )
 
 type errHashMismatch struct {
@@ -37,7 +40,7 @@ func (err *errHashMismatch) Error() string {
 
 // Calculates bitrot in chunks and writes the hash into the stream.
 type streamingBitrotWriter struct {
-	iow       *io.PipeWriter
+	iow       io.WriteCloser
 	h         hash.Hash
 	shardSize int64
 	canClose  chan struct{} // Needed to avoid race explained in Close() call.
@@ -70,19 +73,28 @@ func (b *streamingBitrotWriter) Close() error {
 	return err
 }
 
+var (
+	ioDeadline, _ = time.ParseDuration(env.Get("MINIO_IO_DEADLINE", ""))
+)
+
 // Returns streaming bitrot writer implementation.
-func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
+func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64, heal bool) io.Writer {
 	r, w := io.Pipe()
 	h := algo.New()
-	bw := &streamingBitrotWriter{w, h, shardSize, make(chan struct{})}
+
+	var wc io.WriteCloser = w
+	if ioDeadline > 0 && !heal {
+		wc = xioutil.NewDeadlineWriter(w, ioDeadline)
+	}
+
+	bw := &streamingBitrotWriter{wc, h, shardSize, make(chan struct{})}
 	go func() {
 		totalFileSize := int64(-1) // For compressed objects length will be unknown (represented by length=-1)
 		if length != -1 {
 			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
 			totalFileSize = bitrotSumsTotalSize + length
 		}
-		err := disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r)
-		r.CloseWithError(err)
+		r.CloseWithError(disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r))
 		close(bw.canClose)
 	}()
 	return bw