From 6403c8d5d251dc6aec37d7c16baec55e3a1efecf Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 11 Feb 2025 11:18:59 +0100 Subject: [PATCH] use tsweb debugger (#2420) This PR switches the homegrown debug endpoint to using tsweb.Debugger, a neat toolkit with batteries included for pprof and friends, and making it easy to add additional debug info: I've started out by adding a bunch of "introspect" endpoints image So users can see the acl, filter, config, derpmap and connected nodes as headscale sees them. --- CHANGELOG.md | 10 ++-- flake.nix | 2 +- go.mod | 2 + go.sum | 4 ++ hscontrol/app.go | 17 +------ hscontrol/debug.go | 120 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 hscontrol/debug.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 13cc7fe0..9a1cf1b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,15 @@ ## Next - ### Changes - `oidc.map_legacy_users` and `oidc.strip_email_domain` has been removed [#2411](https://github.com/juanfont/headscale/pull/2411) - +- Add more information to `/debug` endpoint + [#2420](https://github.com/juanfont/headscale/pull/2420) + - It is now possible to inspect running goroutines and take profiles + - View of config, policy, filter, ssh policy per node, connected nodes and + DERPmap ## 0.25.0 (2025-02-xx) @@ -23,7 +26,7 @@ - A logged out node logging in with the same user will replace the existing node. - Remove support for Tailscale clients older than 1.62 (Capability version 87) - [#2405](https://github.com/juanfont/headscale/pull/2405) + [#2405](https://github.com/juanfont/headscale/pull/2405) ### Changes @@ -49,6 +52,7 @@ ## 0.24.3 (2025-02-07) ### Changes + - Fix migration error caused by nodes having invalid auth keys [#2412](https://github.com/juanfont/headscale/pull/2412) - Pre auth keys belonging to a user are no longer deleted with the user diff --git a/flake.nix b/flake.nix index ef2f5974..789133fd 100644 --- a/flake.nix +++ b/flake.nix @@ -30,7 +30,7 @@ # When updating go.mod or go.sum, a new sha will need to be calculated, # update this if you have a mismatch after doing a change to those files. - vendorHash = "sha256-ZQj2A0GdLhHc7JLW7qgpGBveXXNWg9ueSG47OZQQXEw="; + vendorHash = "sha256-CoxqEAxGdefyiIhz84LXXxPrZ1JWsX8Ernv1USr9JTs="; subPackages = ["cmd/headscale"]; diff --git a/go.mod b/go.mod index ecf94318..a5b9de7b 100644 --- a/go.mod +++ b/go.mod @@ -89,6 +89,7 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/akutz/memconn v0.1.0 // indirect github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect + github.com/arl/statsviz v0.6.0 // indirect github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect @@ -141,6 +142,7 @@ require ( github.com/gookit/color v1.5.4 // indirect github.com/gorilla/csrf v1.7.3-0.20250123201450-9dd6af1f6d30 // indirect github.com/gorilla/securecookie v1.1.2 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/hdevalence/ed25519consensus v0.2.0 // indirect github.com/illarion/gonotify/v2 v2.0.3 // indirect diff --git a/go.sum b/go.sum index a6497cb1..88263ed4 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/arl/statsviz v0.6.0 h1:jbW1QJkEYQkufd//4NDYRSNBpwJNrdzPahF7ZmoGdyE= +github.com/arl/statsviz v0.6.0/go.mod h1:0toboo+YGSUXDaS4g1D5TVS4dXs7S7YYT5J/qnW2h8s= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= @@ -242,6 +244,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= diff --git a/hscontrol/app.go b/hscontrol/app.go index 2f1cd4cd..48f375fa 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -36,7 +36,6 @@ import ( "github.com/juanfont/headscale/hscontrol/util" zerolog "github.com/philip-bui/grpc-zerolog" "github.com/pkg/profile" - "github.com/prometheus/client_golang/prometheus/promhttp" zl "github.com/rs/zerolog" "github.com/rs/zerolog/log" "golang.org/x/crypto/acme" @@ -786,26 +785,12 @@ func (h *Headscale) Serve() error { log.Info(). Msgf("listening and serving HTTP on: %s", h.cfg.Addr) - debugMux := http.NewServeMux() - debugMux.Handle("/debug/pprof/", http.DefaultServeMux) - debugMux.HandleFunc("/debug/notifier", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte(h.nodeNotifier.String())) - }) - debugMux.Handle("/metrics", promhttp.Handler()) - - debugHTTPServer := &http.Server{ - Addr: h.cfg.MetricsAddr, - Handler: debugMux, - ReadTimeout: types.HTTPTimeout, - WriteTimeout: 0, - } - debugHTTPListener, err := net.Listen("tcp", h.cfg.MetricsAddr) if err != nil { return fmt.Errorf("failed to bind to TCP address: %w", err) } + debugHTTPServer := h.debugHTTPServer() errorGroup.Go(func() error { return debugHTTPServer.Serve(debugHTTPListener) }) log.Info(). diff --git a/hscontrol/debug.go b/hscontrol/debug.go new file mode 100644 index 00000000..f509a43c --- /dev/null +++ b/hscontrol/debug.go @@ -0,0 +1,120 @@ +package hscontrol + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/arl/statsviz" + "github.com/juanfont/headscale/hscontrol/types" + "github.com/prometheus/client_golang/prometheus/promhttp" + "tailscale.com/tailcfg" + "tailscale.com/tsweb" +) + +func (h *Headscale) debugHTTPServer() *http.Server { + debugMux := http.NewServeMux() + debug := tsweb.Debugger(debugMux) + debug.Handle("notifier", "Connected nodes in notifier", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(h.nodeNotifier.String())) + })) + debug.Handle("config", "Current configuration", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + config, err := json.MarshalIndent(h.cfg, "", " ") + if err != nil { + httpError(w, err) + return + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write(config) + })) + debug.Handle("policy", "Current policy", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pol, err := h.policyBytes() + if err != nil { + httpError(w, err) + return + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write(pol) + })) + debug.Handle("filter", "Current filter", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + filter := h.polMan.Filter() + + filterJSON, err := json.MarshalIndent(filter, "", " ") + if err != nil { + httpError(w, err) + return + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write(filterJSON) + })) + debug.Handle("ssh", "SSH Policy per node", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + nodes, err := h.db.ListNodes() + if err != nil { + httpError(w, err) + return + } + + sshPol := make(map[string]*tailcfg.SSHPolicy) + for _, node := range nodes { + pol, err := h.polMan.SSHPolicy(node) + if err != nil { + httpError(w, err) + return + } + + sshPol[fmt.Sprintf("id:%d hostname:%s givenname:%s", node.ID, node.Hostname, node.GivenName)] = pol + } + + sshJSON, err := json.MarshalIndent(sshPol, "", " ") + if err != nil { + httpError(w, err) + return + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write(sshJSON) + })) + debug.Handle("derpmap", "Current DERPMap", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dm := h.DERPMap + + dmJSON, err := json.MarshalIndent(dm, "", " ") + if err != nil { + httpError(w, err) + return + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write(dmJSON) + })) + debug.Handle("registration-cache", "Pending registrations", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + registrationsJSON, err := json.MarshalIndent(h.registrationCache.Items(), "", " ") + if err != nil { + httpError(w, err) + return + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write(registrationsJSON) + })) + + err := statsviz.Register(debugMux) + if err == nil { + debug.URL("/debug/statsviz", "Statsviz (visualise go metrics)") + } + + debug.URL("/metrics", "Prometheus metrics") + debugMux.Handle("/metrics", promhttp.Handler()) + + debugHTTPServer := &http.Server{ + Addr: h.cfg.MetricsAddr, + Handler: debugMux, + ReadTimeout: types.HTTPTimeout, + WriteTimeout: 0, + } + + return debugHTTPServer +}