add variadic delays in peer notification retries (#17592)

just adds more `jitter` in our retries to avoid
burst flooding for peer calls.

cmd/notification.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"sync"
@@ -87,6 +88,8 @@ func (g *NotificationGroup) Wait() []NotificationPeerErr {
 // The first call to return a non-nil error will be
 // collected in errs slice and returned by Wait().
 func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	g.wg.Add(1)
 
 	go func() {
@@ -103,9 +106,9 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
 					ctx := logger.SetReqInfo(ctx, reqInfo)
 					logger.LogIf(ctx, err)
 				}
-				// Wait for one second and no need wait after last attempt.
+				// Wait for a minimum of 100ms and dynamically increase this based on number of attempts.
 				if i < g.retryCount-1 {
-					time.Sleep(1 * time.Second)
+					time.Sleep(100*time.Millisecond + time.Duration(r.Float64()*float64(time.Second)))
 				}
 				continue
 			}