auth/rpc: Make auth rpc client retry configurable. (#3695)

Currently the auth rpc client defaults to to a maximum
cap of 30seconds timeout. Make this to be configurable
by the caller of authRPCClient during initialization, if no
such config is provided then default to 30 seconds.

cmd/retry.go

@@ -48,7 +48,8 @@ func (r *lockedRandSource) Seed(seed int64) {
 // MaxJitter will randomize over the full exponential backoff time
 const MaxJitter = 1.0
 
-// NoJitter disables the use of jitter for randomizing the exponential backoff time
+// NoJitter disables the use of jitter for randomizing the
+// exponential backoff time
 const NoJitter = 0.0
 
 // Global random source for fetching random values.
@@ -56,9 +57,11 @@ var globalRandomSource = rand.New(&lockedRandSource{
 	src: rand.NewSource(time.Now().UTC().UnixNano()),
 })
 
-// newRetryTimer creates a timer with exponentially increasing delays
-// until the maximum retry attempts are reached.
-func newRetryTimer(unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
+// newRetryTimerJitter creates a timer with exponentially increasing delays
+// until the maximum retry attempts are reached. - this function is a fully
+// configurable version, meant for only advanced use cases. For the most part
+// one should use newRetryTimerSimple and newRetryTimer.
+func newRetryTimerWithJitter(unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
 	attemptCh := make(chan int)
 
 	// normalize jitter to the range [0, 1.0]
@@ -113,5 +116,27 @@ func newRetryTimer(unit time.Duration, cap time.Duration, jitter float64, doneCh
 
 		}
 	}()
+
+	// Start reading..
 	return attemptCh
 }
+
+// Default retry constants.
+var (
+	defaultRetryUnit = time.Second      // 1 second.
+	defaultRetryCap  = 30 * time.Second // 30 seconds.
+)
+
+// newRetryTimer creates a timer with exponentially increasing delays
+// until the maximum retry attempts are reached. - this function provides
+// resulting retry values to be of maximum jitter.
+func newRetryTimer(unit time.Duration, cap time.Duration, doneCh chan struct{}) <-chan int {
+	return newRetryTimerWithJitter(unit, cap, MaxJitter, doneCh)
+}
+
+// newRetryTimerSimple creates a timer with exponentially increasing delays
+// until the maximum retry attempts are reached. - this function is a
+// simpler version with all default values.
+func newRetryTimerSimple(doneCh chan struct{}) <-chan int {
+	return newRetryTimerWithJitter(defaultRetryUnit, defaultRetryCap, MaxJitter, doneCh)
+}