From f41edb23e2cd5d68e468fb79e6aeb9162b541407 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 7 Jul 2023 07:47:38 -0700 Subject: [PATCH] add variadic delays in peer notification retries (#17592) just adds more `jitter` in our retries to avoid burst flooding for peer calls. --- cmd/notification.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/notification.go b/cmd/notification.go index dcbe4263d..b587cd0e5 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io" + "math/rand" "net/http" "net/url" "sync" @@ -87,6 +88,8 @@ func (g *NotificationGroup) Wait() []NotificationPeerErr { // The first call to return a non-nil error will be // collected in errs slice and returned by Wait(). func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + g.wg.Add(1) go func() { @@ -103,9 +106,9 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a ctx := logger.SetReqInfo(ctx, reqInfo) logger.LogIf(ctx, err) } - // Wait for one second and no need wait after last attempt. + // Wait for a minimum of 100ms and dynamically increase this based on number of attempts. if i < g.retryCount-1 { - time.Sleep(1 * time.Second) + time.Sleep(100*time.Millisecond + time.Duration(r.Float64()*float64(time.Second))) } continue }