crypto: reduce retry delay when retrying KES requests (#10394)

This commit reduces the retry delay when retrying a request
to a KES server by:
 - reducing the max. jitter delay from 3s to 1.5s
 - skipping the random delay when there are more KES endpoints
   available.

If there are more KES endpoints we can directly retry to the request
by sending it to the next endpoint - as pointed out by @krishnasrinivas

cmd/crypto/kes.go

@@ -414,19 +414,22 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
 			return response, nil
 		}
 
+		// If the error is not temp. / retryable => fail the request immediately.
 		if !xnet.IsNetworkOrHostDown(err) &&
 			!errors.Is(err, io.EOF) &&
 			!errors.Is(err, io.ErrUnexpectedEOF) &&
 			!errors.Is(err, context.DeadlineExceeded) {
 			return nil, err
 		}
-
-		// retriable network errors.
-		remain := retryMax - i
-		if remain <= 0 {
+		if remain := retryMax - i; remain <= 0 { // Fail if we exceeded our retry limit.
 			return response, err
 		}
 
+		// If there are more KES instances then skip waiting and
+		// try the next endpoint directly.
+		if i < len(c.endpoints) {
+			continue
+		}
 		<-time.After(LinearJitterBackoff(retryWaitMin, retryWaitMax, i))
 	}
 }

cmd/crypto/retry.go

@@ -21,8 +21,8 @@ import (
 
 // default retry configuration
 const (
-	retryWaitMin = 500 * time.Millisecond // minimum retry limit.
-	retryWaitMax = 3 * time.Second        // 3 secs worth of max retry.
+	retryWaitMin = 100 * time.Millisecond  // minimum retry limit.
+	retryWaitMax = 1500 * time.Millisecond // 1.5 secs worth of max retry.
 )
 
 // LinearJitterBackoff provides the time.Duration for a caller to