Fix delays and iterim fix for the partial fix in #3502 (#3511)

This patch uses a technique where in a retryable storage
before object layer initialization has a higher delay
and waits for longer period upto 4 times with time unit
of seconds.

And uses another set of configuration after the disks
have been formatted, i.e use a lower retry backoff rate
and retrying only once per 5 millisecond.

Network IO error count is reduced to a lower value i.e 256
before we reject the disk completely. This is done so that
combination of retry logic and total error count roughly
come to around 2.5secs which is when we basically take the
disk offline completely.

NOTE: This patch doesn't fix the issue of what if the disk
is completely dead and comes back again after the initialization.
Such a mutating state requires a change in our startup sequence
which will be done subsequently. This is an interim fix to alleviate
users from these issues.

cmd/retry-storage.go

@@ -22,12 +22,25 @@ import (
 	"github.com/minio/minio/pkg/disk"
 )
 
+const (
+	// Attempt to retry only this many number of times before
+	// giving up on the remote disk entirely during initialization.
+	globalStorageInitRetryThreshold = 4
+
+	// Attempt to retry only this many number of times before
+	// giving up on the remote disk entirely after initialization.
+	globalStorageRetryThreshold = 1
+)
+
 // Retry storage is an instance of StorageAPI which
 // additionally verifies upon network shutdown if the
 // underlying storage is available and is really
 // formatted.
 type retryStorage struct {
-	remoteStorage StorageAPI
+	remoteStorage    StorageAPI
+	maxRetryAttempts int
+	retryUnit        time.Duration
+	retryCap         time.Duration
 }
 
 // String representation of remoteStorage.
@@ -209,13 +222,13 @@ func (f retryStorage) reInit() (err error) {
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
-	for i := range newRetryTimer(time.Second, time.Second*30, MaxJitter, doneCh) {
+	for i := range newRetryTimer(f.retryUnit, f.retryCap, MaxJitter, doneCh) {
 		// Initialize and make a new login attempt.
 		err = f.remoteStorage.Init()
 		if err != nil {
 			// No need to return error until the retry count
 			// threshold has reached.
-			if i < globalMaxStorageRetryThreshold {
+			if i < f.maxRetryAttempts {
 				continue
 			}
 			return err
@@ -226,7 +239,7 @@ func (f retryStorage) reInit() (err error) {
 		if err != nil {
 			// No need to return error until the retry count
 			// threshold has reached.
-			if i < globalMaxStorageRetryThreshold {
+			if i < f.maxRetryAttempts {
 				continue
 			}
 			return err