[feat] Add configurable deadline for writers (#11822)

This PR adds deadlines per Write() calls, such
that slow drives are timed-out appropriately and
the overall responsiveness for Writes() is always
up to a predefined threshold providing applications
sustained latency even if one of the drives is slow
to respond.

pkg/ioutil/ioutil_test.go

@@ -18,12 +18,49 @@ package ioutil
 
 import (
 	"bytes"
+	"context"
 	"io"
 	goioutil "io/ioutil"
 	"os"
 	"testing"
+	"time"
 )
 
+type sleepWriter struct {
+	timeout time.Duration
+}
+
+func (w *sleepWriter) Write(p []byte) (n int, err error) {
+	time.Sleep(w.timeout)
+	return len(p), nil
+}
+
+func (w *sleepWriter) Close() error {
+	return nil
+}
+
+func TestDeadlineWriter(t *testing.T) {
+	w := NewDeadlineWriter(&sleepWriter{timeout: 500 * time.Millisecond}, 450*time.Millisecond)
+	_, err := w.Write([]byte("1"))
+	w.Close()
+	if err != context.Canceled {
+		t.Error("DeadlineWriter shouldn't be successful - should return context.Canceled")
+	}
+	_, err = w.Write([]byte("1"))
+	if err != context.Canceled {
+		t.Error("DeadlineWriter shouldn't be successful - should return context.Canceled")
+	}
+	w = NewDeadlineWriter(&sleepWriter{timeout: 500 * time.Millisecond}, 600*time.Millisecond)
+	n, err := w.Write([]byte("abcd"))
+	w.Close()
+	if err != nil {
+		t.Errorf("DeadlineWriter should succeed but failed with %s", err)
+	}
+	if n != 4 {
+		t.Errorf("DeadlineWriter should succeed but should have only written 0 bytes, but returned %d instead", n)
+	}
+}
+
 func TestCloseOnWriter(t *testing.T) {
 	writer := WriteOnClose(goioutil.Discard)
 	if writer.HasWritten() {