Faster startup of large distributed systems with latency (#16259)

cmd/server-main.go

@@ -358,6 +358,12 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	lockTimeout := newDynamicTimeout(5*time.Second, 3*time.Second)
+	// do not retry to avoid high contention on startup.
+	lockTimeout.retryInterval = -1
+
+	// Do an initial random sleep to avoid stampeding herd of initial
+	// lock request. This will spread locks requests over 1 second.
+	time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
 
 	for {
 		select {
@@ -380,7 +386,8 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 		if err != nil {
 			logger.Info("Waiting for all MinIO sub-systems to be initialized.. trying to acquire lock")
 
-			time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
+			// Sleep 0 -> 2 seconds to average 1 second retry interval.
+			time.Sleep(time.Duration(r.Float64() * 2 * float64(time.Second)))
 			continue
 		}