From 3cf8a7c888c3e305fbaccc0bfae298427bf962a3 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 29 Apr 2024 10:39:04 -0700 Subject: [PATCH] Always unfreeze when connection dies (#19634) Unfreeze as soon as the incoming connection is terminated and don't wait for everything to complete. We don't want to keep the services frozen if something becomes stuck. --- cmd/admin-handlers.go | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index f3eb4211f..9918c273c 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1576,7 +1576,8 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request) // NetperfHandler - perform mesh style network throughput test func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction) if objectAPI == nil { @@ -1597,6 +1598,15 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) ctx = lkctx.Context() defer nsLock.Unlock(lkctx) + // Freeze all incoming S3 API calls before running speedtest. + globalNotificationSys.ServiceFreeze(ctx, true) + + // Unfreeze as soon as request context is canceled or when the function returns. + go func() { + <-ctx.Done() + globalNotificationSys.ServiceFreeze(ctx, false) + }() + durationStr := r.Form.Get(peerRESTDuration) duration, err := time.ParseDuration(durationStr) if err != nil { @@ -1622,7 +1632,8 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) // increasing concurrency and stopping when we have reached the limits on the // system. func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction) if objectAPI == nil { @@ -1692,8 +1703,11 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http. // Freeze all incoming S3 API calls before running speedtest. globalNotificationSys.ServiceFreeze(ctx, true) - // unfreeze all incoming S3 API calls after speedtest. - defer globalNotificationSys.ServiceFreeze(ctx, false) + // Unfreeze as soon as request context is canceled or when the function returns. + go func() { + <-ctx.Done() + globalNotificationSys.ServiceFreeze(ctx, false) + }() keepAliveTicker := time.NewTicker(500 * time.Millisecond) defer keepAliveTicker.Stop() @@ -1793,7 +1807,8 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo, // DriveSpeedtestHandler - reports throughput of drives available in the cluster func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction) if objectAPI == nil { @@ -1803,8 +1818,11 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R // Freeze all incoming S3 API calls before running speedtest. globalNotificationSys.ServiceFreeze(ctx, true) - // unfreeze all incoming S3 API calls after speedtest. - defer globalNotificationSys.ServiceFreeze(ctx, false) + // Unfreeze as soon as request context is canceled or when the function returns. + go func() { + <-ctx.Done() + globalNotificationSys.ServiceFreeze(ctx, false) + }() serial := r.Form.Get("serial") == "true" blockSizeStr := r.Form.Get("blocksize")