Add support profile deadlines and concurrent operations (#20244)

* Allow a maximum of 10 seconds to start profiling operations.
* Download up to 16 profiles concurrently, but only allow 10 seconds for
  each (does not include write time).
* Add cluster info as the first operation.
* Ignore remote download errors.
* Stop remote profiles if the request is terminated.
This commit is contained in:
Klaus Post
2024-08-15 03:36:00 -07:00
committed by GitHub
parent b508264ac4
commit d96798ae7b
3 changed files with 55 additions and 37 deletions

View File

@@ -1036,7 +1036,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
// Start profiling on remote servers.
var hostErrs []NotificationPeerErr
for _, profiler := range profiles {
hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(profiler)...)
hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(ctx, profiler)...)
// Start profiling locally as well.
prof, err := startProfiler(profiler)
@@ -1117,7 +1117,11 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
// Start profiling on remote servers.
for _, profiler := range profiles {
globalNotificationSys.StartProfiling(profiler)
// Limit start time to max 10s.
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
globalNotificationSys.StartProfiling(ctx, profiler)
// StartProfiling blocks, so we can cancel now.
cancel()
// Start profiling locally as well.
prof, err := startProfiler(profiler)
@@ -1132,6 +1136,10 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
for {
select {
case <-ctx.Done():
// Stop remote profiles
go globalNotificationSys.DownloadProfilingData(GlobalContext, io.Discard)
// Stop local
globalProfilerMu.Lock()
defer globalProfilerMu.Unlock()
for k, v := range globalProfiler {