[feat] Add configurable deadline for writers (#11822)

This PR adds deadlines per Write() calls, such
that slow drives are timed-out appropriately and
the overall responsiveness for Writes() is always
up to a predefined threshold providing applications
sustained latency even if one of the drives is slow
to respond.

pkg/ioutil/ioutil.go

@@ -20,8 +20,10 @@ package ioutil
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"os"
+	"time"
 
 	"github.com/minio/minio/pkg/disk"
 )
@@ -64,6 +66,56 @@ func WriteOnClose(w io.Writer) *WriteOnCloser {
 	return &WriteOnCloser{w, false}
 }
 
+type ioret struct {
+	n   int
+	err error
+}
+
+// DeadlineWriter deadline writer with context
+type DeadlineWriter struct {
+	io.WriteCloser
+	timeout time.Duration
+	err     error
+}
+
+// NewDeadlineWriter wraps a writer to make it respect given deadline
+// value per Write(). If there is a blocking write, the returned Writer
+// will return whenever the timer hits (the return values are n=0
+// and err=context.Canceled.)
+func NewDeadlineWriter(w io.WriteCloser, timeout time.Duration) io.WriteCloser {
+	return &DeadlineWriter{WriteCloser: w, timeout: timeout}
+}
+
+func (w *DeadlineWriter) Write(buf []byte) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	c := make(chan ioret, 1)
+	t := time.NewTimer(w.timeout)
+	defer t.Stop()
+
+	go func() {
+		n, err := w.WriteCloser.Write(buf)
+		c <- ioret{n, err}
+		close(c)
+	}()
+
+	select {
+	case r := <-c:
+		w.err = r.err
+		return r.n, r.err
+	case <-t.C:
+		w.err = context.Canceled
+		return 0, context.Canceled
+	}
+}
+
+// Close closer interface to close the underlying closer
+func (w *DeadlineWriter) Close() error {
+	return w.WriteCloser.Close()
+}
+
 // LimitWriter implements io.WriteCloser.
 //
 // This is implemented such that we want to restrict

pkg/ioutil/ioutil_test.go

@@ -18,12 +18,49 @@ package ioutil
 
 import (
 	"bytes"
+	"context"
 	"io"
 	goioutil "io/ioutil"
 	"os"
 	"testing"
+	"time"
 )
 
+type sleepWriter struct {
+	timeout time.Duration
+}
+
+func (w *sleepWriter) Write(p []byte) (n int, err error) {
+	time.Sleep(w.timeout)
+	return len(p), nil
+}
+
+func (w *sleepWriter) Close() error {
+	return nil
+}
+
+func TestDeadlineWriter(t *testing.T) {
+	w := NewDeadlineWriter(&sleepWriter{timeout: 500 * time.Millisecond}, 450*time.Millisecond)
+	_, err := w.Write([]byte("1"))
+	w.Close()
+	if err != context.Canceled {
+		t.Error("DeadlineWriter shouldn't be successful - should return context.Canceled")
+	}
+	_, err = w.Write([]byte("1"))
+	if err != context.Canceled {
+		t.Error("DeadlineWriter shouldn't be successful - should return context.Canceled")
+	}
+	w = NewDeadlineWriter(&sleepWriter{timeout: 500 * time.Millisecond}, 600*time.Millisecond)
+	n, err := w.Write([]byte("abcd"))
+	w.Close()
+	if err != nil {
+		t.Errorf("DeadlineWriter should succeed but failed with %s", err)
+	}
+	if n != 4 {
+		t.Errorf("DeadlineWriter should succeed but should have only written 0 bytes, but returned %d instead", n)
+	}
+}
+
 func TestCloseOnWriter(t *testing.T) {
 	writer := WriteOnClose(goioutil.Discard)
 	if writer.HasWritten() {