From 00e7550e760b2d3d759471ff55d2b6e2dc81ad2b Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 9 Feb 2024 07:26:41 +0100 Subject: [PATCH] Add assert func for verifying status, netmap and netcheck (#1723) --- ...egration-v2-TestPingAllByIPPublicDERP.yaml | 67 ++++++++ hscontrol/poll.go | 10 +- hscontrol/types/node_test.go | 108 +++++++++++++ hscontrol/util/test.go | 6 +- integration/auth_oidc_test.go | 4 + integration/auth_web_flow_test.go | 4 + integration/embedded_derp_test.go | 29 ++-- integration/general_test.go | 140 ++++++++++++++++- integration/tailscale.go | 2 + integration/tsic/tsic.go | 34 ++++ integration/utils.go | 148 +++++++++++++++++- 11 files changed, 534 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/test-integration-v2-TestPingAllByIPPublicDERP.yaml diff --git a/.github/workflows/test-integration-v2-TestPingAllByIPPublicDERP.yaml b/.github/workflows/test-integration-v2-TestPingAllByIPPublicDERP.yaml new file mode 100644 index 00000000..18fd3417 --- /dev/null +++ b/.github/workflows/test-integration-v2-TestPingAllByIPPublicDERP.yaml @@ -0,0 +1,67 @@ +# DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go +# To regenerate, run "go generate" in cmd/gh-action-integration-generator/ + +name: Integration Test v2 - TestPingAllByIPPublicDERP + +on: [pull_request] + +concurrency: + group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + TestPingAllByIPPublicDERP: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - uses: satackey/action-docker-layer-caching@main + continue-on-error: true + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v34 + with: + files: | + *.nix + go.* + **/*.go + integration_test/ + config-example.yaml + + - name: Run TestPingAllByIPPublicDERP + uses: Wandalen/wretry.action@master + if: steps.changed-files.outputs.any_changed == 'true' + with: + attempt_limit: 5 + command: | + nix develop --command -- docker run \ + --tty --rm \ + --volume ~/.cache/hs-integration-go:/go \ + --name headscale-test-suite \ + --volume $PWD:$PWD -w $PWD/integration \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume $PWD/control_logs:/tmp/control \ + golang:1 \ + go run gotest.tools/gotestsum@latest -- ./... \ + -failfast \ + -timeout 120m \ + -parallel 1 \ + -run "^TestPingAllByIPPublicDERP$" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: logs + path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/hscontrol/poll.go b/hscontrol/poll.go index f00152d4..03f52ed9 100644 --- a/hscontrol/poll.go +++ b/hscontrol/poll.go @@ -201,9 +201,15 @@ func (h *Headscale) handlePoll( return } + // TODO(kradalby): Figure out why patch changes does + // not show up in output from `tailscale debug netmap`. + // stateUpdate := types.StateUpdate{ + // Type: types.StatePeerChangedPatch, + // ChangePatches: []*tailcfg.PeerChange{&change}, + // } stateUpdate := types.StateUpdate{ - Type: types.StatePeerChangedPatch, - ChangePatches: []*tailcfg.PeerChange{&change}, + Type: types.StatePeerChanged, + ChangeNodes: types.Nodes{node}, } if stateUpdate.Valid() { ctx := types.NotifyCtx(context.Background(), "poll-nodeupdate-peers-patch", node.Hostname) diff --git a/hscontrol/types/node_test.go b/hscontrol/types/node_test.go index 7e6c9840..712a839e 100644 --- a/hscontrol/types/node_test.go +++ b/hscontrol/types/node_test.go @@ -6,6 +6,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/juanfont/headscale/hscontrol/util" "tailscale.com/tailcfg" "tailscale.com/types/key" ) @@ -366,3 +367,110 @@ func TestPeerChangeFromMapRequest(t *testing.T) { }) } } + +func TestApplyPeerChange(t *testing.T) { + tests := []struct { + name string + nodeBefore Node + change *tailcfg.PeerChange + want Node + }{ + { + name: "hostinfo-and-netinfo-not-exists", + nodeBefore: Node{}, + change: &tailcfg.PeerChange{ + DERPRegion: 1, + }, + want: Node{ + Hostinfo: &tailcfg.Hostinfo{ + NetInfo: &tailcfg.NetInfo{ + PreferredDERP: 1, + }, + }, + }, + }, + { + name: "hostinfo-netinfo-not-exists", + nodeBefore: Node{ + Hostinfo: &tailcfg.Hostinfo{ + Hostname: "test", + }, + }, + change: &tailcfg.PeerChange{ + DERPRegion: 3, + }, + want: Node{ + Hostinfo: &tailcfg.Hostinfo{ + Hostname: "test", + NetInfo: &tailcfg.NetInfo{ + PreferredDERP: 3, + }, + }, + }, + }, + { + name: "hostinfo-netinfo-exists-derp-set", + nodeBefore: Node{ + Hostinfo: &tailcfg.Hostinfo{ + Hostname: "test", + NetInfo: &tailcfg.NetInfo{ + PreferredDERP: 999, + }, + }, + }, + change: &tailcfg.PeerChange{ + DERPRegion: 2, + }, + want: Node{ + Hostinfo: &tailcfg.Hostinfo{ + Hostname: "test", + NetInfo: &tailcfg.NetInfo{ + PreferredDERP: 2, + }, + }, + }, + }, + { + name: "endpoints-not-set", + nodeBefore: Node{}, + change: &tailcfg.PeerChange{ + Endpoints: []netip.AddrPort{ + netip.MustParseAddrPort("8.8.8.8:88"), + }, + }, + want: Node{ + Endpoints: []netip.AddrPort{ + netip.MustParseAddrPort("8.8.8.8:88"), + }, + }, + }, + { + name: "endpoints-set", + nodeBefore: Node{ + Endpoints: []netip.AddrPort{ + netip.MustParseAddrPort("6.6.6.6:66"), + }, + }, + change: &tailcfg.PeerChange{ + Endpoints: []netip.AddrPort{ + netip.MustParseAddrPort("8.8.8.8:88"), + }, + }, + want: Node{ + Endpoints: []netip.AddrPort{ + netip.MustParseAddrPort("8.8.8.8:88"), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.nodeBefore.ApplyPeerChange(tt.change) + + if diff := cmp.Diff(tt.want, tt.nodeBefore, util.Comparers...); diff != "" { + t.Errorf("Patch unexpected result (-want +got):\n%s", diff) + } + }) + } +} diff --git a/hscontrol/util/test.go b/hscontrol/util/test.go index 6d465426..0a23acb4 100644 --- a/hscontrol/util/test.go +++ b/hscontrol/util/test.go @@ -15,6 +15,10 @@ var IPComparer = cmp.Comparer(func(x, y netip.Addr) bool { return x.Compare(y) == 0 }) +var AddrPortComparer = cmp.Comparer(func(x, y netip.AddrPort) bool { + return x == y +}) + var MkeyComparer = cmp.Comparer(func(x, y key.MachinePublic) bool { return x.String() == y.String() }) @@ -28,5 +32,5 @@ var DkeyComparer = cmp.Comparer(func(x, y key.DiscoPublic) bool { }) var Comparers []cmp.Option = []cmp.Option{ - IPComparer, PrefixComparer, MkeyComparer, NkeyComparer, DkeyComparer, + IPComparer, PrefixComparer, AddrPortComparer, MkeyComparer, NkeyComparer, DkeyComparer, } diff --git a/integration/auth_oidc_test.go b/integration/auth_oidc_test.go index 7a0ed9c7..36e74a8d 100644 --- a/integration/auth_oidc_test.go +++ b/integration/auth_oidc_test.go @@ -83,6 +83,8 @@ func TestOIDCAuthenticationPingAll(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { return x.String() }) @@ -140,6 +142,8 @@ func TestOIDCExpireNodesBasedOnTokenExpiry(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { return x.String() }) diff --git a/integration/auth_web_flow_test.go b/integration/auth_web_flow_test.go index 90ce571b..aa589fac 100644 --- a/integration/auth_web_flow_test.go +++ b/integration/auth_web_flow_test.go @@ -53,6 +53,8 @@ func TestAuthWebFlowAuthenticationPingAll(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { return x.String() }) @@ -90,6 +92,8 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { return x.String() }) diff --git a/integration/embedded_derp_test.go b/integration/embedded_derp_test.go index 3a407496..15ab7add 100644 --- a/integration/embedded_derp_test.go +++ b/integration/embedded_derp_test.go @@ -33,20 +33,23 @@ func TestDERPServerScenario(t *testing.T) { defer scenario.Shutdown() spec := map[string]int{ - "user1": len(MustTestVersions), + "user1": 10, + // "user1": len(MustTestVersions), } - headscaleConfig := map[string]string{} - headscaleConfig["HEADSCALE_DERP_URLS"] = "" - headscaleConfig["HEADSCALE_DERP_SERVER_ENABLED"] = "true" - headscaleConfig["HEADSCALE_DERP_SERVER_REGION_ID"] = "999" - headscaleConfig["HEADSCALE_DERP_SERVER_REGION_CODE"] = "headscale" - headscaleConfig["HEADSCALE_DERP_SERVER_REGION_NAME"] = "Headscale Embedded DERP" - headscaleConfig["HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR"] = "0.0.0.0:3478" - headscaleConfig["HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH"] = "/tmp/derp.key" - // Envknob for enabling DERP debug logs - headscaleConfig["DERP_DEBUG_LOGS"] = "true" - headscaleConfig["DERP_PROBER_DEBUG_LOGS"] = "true" + headscaleConfig := map[string]string{ + "HEADSCALE_DERP_URLS": "", + "HEADSCALE_DERP_SERVER_ENABLED": "true", + "HEADSCALE_DERP_SERVER_REGION_ID": "999", + "HEADSCALE_DERP_SERVER_REGION_CODE": "headscale", + "HEADSCALE_DERP_SERVER_REGION_NAME": "Headscale Embedded DERP", + "HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR": "0.0.0.0:3478", + "HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH": "/tmp/derp.key", + + // Envknob for enabling DERP debug logs + "DERP_DEBUG_LOGS": "true", + "DERP_PROBER_DEBUG_LOGS": "true", + } err = scenario.CreateHeadscaleEnv( spec, @@ -67,6 +70,8 @@ func TestDERPServerScenario(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allHostnames, err := scenario.ListTailscaleClientsFQDNs() assertNoErrListFQDN(t, err) diff --git a/integration/general_test.go b/integration/general_test.go index 5c98cd26..9aae26fc 100644 --- a/integration/general_test.go +++ b/integration/general_test.go @@ -33,7 +33,27 @@ func TestPingAllByIP(t *testing.T) { "user2": len(MustTestVersions), } - err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("pingallbyip")) + headscaleConfig := map[string]string{ + "HEADSCALE_DERP_URLS": "", + "HEADSCALE_DERP_SERVER_ENABLED": "true", + "HEADSCALE_DERP_SERVER_REGION_ID": "999", + "HEADSCALE_DERP_SERVER_REGION_CODE": "headscale", + "HEADSCALE_DERP_SERVER_REGION_NAME": "Headscale Embedded DERP", + "HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR": "0.0.0.0:3478", + "HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH": "/tmp/derp.key", + + // Envknob for enabling DERP debug logs + "DERP_DEBUG_LOGS": "true", + "DERP_PROBER_DEBUG_LOGS": "true", + } + + err = scenario.CreateHeadscaleEnv(spec, + []tsic.Option{}, + hsic.WithTestName("pingallbyip"), + hsic.WithConfigEnv(headscaleConfig), + hsic.WithTLS(), + hsic.WithHostnameAsServerURL(), + ) assertNoErrHeadscaleEnv(t, err) allClients, err := scenario.ListTailscaleClients() @@ -45,6 +65,46 @@ func TestPingAllByIP(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { + return x.String() + }) + + success := pingAllHelper(t, allClients, allAddrs) + t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps)) +} + +func TestPingAllByIPPublicDERP(t *testing.T) { + IntegrationSkip(t) + t.Parallel() + + scenario, err := NewScenario() + assertNoErr(t, err) + defer scenario.Shutdown() + + spec := map[string]int{ + "user1": len(MustTestVersions), + "user2": len(MustTestVersions), + } + + err = scenario.CreateHeadscaleEnv(spec, + []tsic.Option{}, + hsic.WithTestName("pingallbyippubderp"), + ) + assertNoErrHeadscaleEnv(t, err) + + allClients, err := scenario.ListTailscaleClients() + assertNoErrListClients(t, err) + + allIps, err := scenario.ListTailscaleClientsIPs() + assertNoErrListClientIPs(t, err) + + err = scenario.WaitForTailscaleSync() + assertNoErrSync(t, err) + + assertClientsState(t, allClients) + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { return x.String() }) @@ -75,6 +135,8 @@ func TestAuthKeyLogoutAndRelogin(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + clientIPs := make(map[TailscaleClient][]netip.Addr) for _, client := range allClients { ips, err := client.IPs() @@ -114,6 +176,8 @@ func TestAuthKeyLogoutAndRelogin(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allClients, err = scenario.ListTailscaleClients() assertNoErrListClients(t, err) @@ -265,6 +329,8 @@ func TestPingAllByHostname(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allHostnames, err := scenario.ListTailscaleClientsFQDNs() assertNoErrListFQDN(t, err) @@ -451,6 +517,74 @@ func TestTaildrop(t *testing.T) { } } +func TestResolveMagicDNS(t *testing.T) { + IntegrationSkip(t) + t.Parallel() + + scenario, err := NewScenario() + assertNoErr(t, err) + defer scenario.Shutdown() + + spec := map[string]int{ + "magicdns1": len(MustTestVersions), + "magicdns2": len(MustTestVersions), + } + + err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("magicdns")) + assertNoErrHeadscaleEnv(t, err) + + allClients, err := scenario.ListTailscaleClients() + assertNoErrListClients(t, err) + + err = scenario.WaitForTailscaleSync() + assertNoErrSync(t, err) + + assertClientsState(t, allClients) + + // Poor mans cache + _, err = scenario.ListTailscaleClientsFQDNs() + assertNoErrListFQDN(t, err) + + _, err = scenario.ListTailscaleClientsIPs() + assertNoErrListClientIPs(t, err) + + for _, client := range allClients { + for _, peer := range allClients { + // It is safe to ignore this error as we handled it when caching it + peerFQDN, _ := peer.FQDN() + + command := []string{ + "tailscale", + "ip", peerFQDN, + } + result, _, err := client.Execute(command) + if err != nil { + t.Fatalf( + "failed to execute resolve/ip command %s from %s: %s", + peerFQDN, + client.Hostname(), + err, + ) + } + + ips, err := peer.IPs() + if err != nil { + t.Fatalf( + "failed to get ips for %s: %s", + peer.Hostname(), + err, + ) + } + + for _, ip := range ips { + if !strings.Contains(result, ip.String()) { + t.Fatalf("ip %s is not found in \n%s\n", ip.String(), result) + } + } + } + } +} + func TestExpireNode(t *testing.T) { IntegrationSkip(t) t.Parallel() @@ -475,6 +609,8 @@ func TestExpireNode(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { return x.String() }) @@ -599,6 +735,8 @@ func TestNodeOnlineLastSeenStatus(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + assertClientsState(t, allClients) + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { return x.String() }) diff --git a/integration/tailscale.go b/integration/tailscale.go index 7187a812..9d6796bd 100644 --- a/integration/tailscale.go +++ b/integration/tailscale.go @@ -7,6 +7,7 @@ import ( "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/tsic" "tailscale.com/ipn/ipnstate" + "tailscale.com/net/netcheck" "tailscale.com/types/netmap" ) @@ -28,6 +29,7 @@ type TailscaleClient interface { FQDN() (string, error) Status() (*ipnstate.Status, error) Netmap() (*netmap.NetworkMap, error) + Netcheck() (*netcheck.Report, error) WaitForNeedsLogin() error WaitForRunning() error WaitForPeers(expected int) error diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index c30118dd..854d5a71 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -17,6 +17,7 @@ import ( "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" "tailscale.com/ipn/ipnstate" + "tailscale.com/net/netcheck" "tailscale.com/types/netmap" ) @@ -544,6 +545,29 @@ func (t *TailscaleInContainer) Netmap() (*netmap.NetworkMap, error) { return &nm, err } +// Netcheck returns the current Netcheck Report (netcheck.Report) of the Tailscale instance. +func (t *TailscaleInContainer) Netcheck() (*netcheck.Report, error) { + command := []string{ + "tailscale", + "netcheck", + "--format=json", + } + + result, stderr, err := t.Execute(command) + if err != nil { + fmt.Printf("stderr: %s\n", stderr) + return nil, fmt.Errorf("failed to execute tailscale debug netcheck command: %w", err) + } + + var nm netcheck.Report + err = json.Unmarshal([]byte(result), &nm) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal tailscale netcheck: %w", err) + } + + return &nm, err +} + // FQDN returns the FQDN as a string of the Tailscale instance. func (t *TailscaleInContainer) FQDN() (string, error) { if t.fqdn != "" { @@ -648,12 +672,22 @@ func (t *TailscaleInContainer) WaitForPeers(expected int) error { len(peers), ) } else { + // Verify that the peers of a given node is Online + // has a hostname and a DERP relay. for _, peerKey := range peers { peer := status.Peer[peerKey] if !peer.Online { return fmt.Errorf("[%s] peer count correct, but %s is not online", t.hostname, peer.HostName) } + + if peer.HostName == "" { + return fmt.Errorf("[%s] peer count correct, but %s does not have a Hostname", t.hostname, peer.HostName) + } + + if peer.Relay == "" { + return fmt.Errorf("[%s] peer count correct, but %s does not have a DERP", t.hostname, peer.HostName) + } } } diff --git a/integration/utils.go b/integration/utils.go index e17e18a2..ae4441b8 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -7,6 +7,8 @@ import ( "time" "github.com/juanfont/headscale/integration/tsic" + "github.com/stretchr/testify/assert" + "tailscale.com/util/cmpver" ) const ( @@ -83,7 +85,7 @@ func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string, opts for _, addr := range addrs { err := client.Ping(addr, opts...) if err != nil { - t.Fatalf("failed to ping %s from %s: %s", addr, client.Hostname(), err) + t.Errorf("failed to ping %s from %s: %s", addr, client.Hostname(), err) } else { success++ } @@ -120,6 +122,148 @@ func pingDerpAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) return success } +// assertClientsState validates the status and netmap of a list of +// clients for the general case of all to all connectivity. +func assertClientsState(t *testing.T, clients []TailscaleClient) { + t.Helper() + + for _, client := range clients { + assertValidStatus(t, client) + assertValidNetmap(t, client) + assertValidNetcheck(t, client) + } +} + +// assertValidNetmap asserts that the netmap of a client has all +// the minimum required fields set to a known working config for +// the general case. Fields are checked on self, then all peers. +// This test is not suitable for ACL/partial connection tests. +// This test can only be run on clients from 1.56.1. It will +// automatically pass all clients below that and is safe to call +// for all versions. +func assertValidNetmap(t *testing.T, client TailscaleClient) { + t.Helper() + + if cmpver.Compare("1.56.1", client.Version()) <= 0 || + !strings.Contains(client.Hostname(), "unstable") || + !strings.Contains(client.Hostname(), "head") { + return + } + + netmap, err := client.Netmap() + if err != nil { + t.Fatalf("getting netmap for %q: %s", client.Hostname(), err) + } + + assert.Truef(t, netmap.SelfNode.Hostinfo().Valid(), "%q does not have Hostinfo", client.Hostname()) + if hi := netmap.SelfNode.Hostinfo(); hi.Valid() { + assert.LessOrEqual(t, 1, netmap.SelfNode.Hostinfo().Services().Len(), "%q does not have enough services, got: %v", client.Hostname(), netmap.SelfNode.Hostinfo().Services()) + } + + assert.NotEmptyf(t, netmap.SelfNode.AllowedIPs(), "%q does not have any allowed IPs", client.Hostname()) + assert.NotEmptyf(t, netmap.SelfNode.Addresses(), "%q does not have any addresses", client.Hostname()) + + assert.Truef(t, *netmap.SelfNode.Online(), "%q is not online", client.Hostname()) + + assert.Falsef(t, netmap.SelfNode.Key().IsZero(), "%q does not have a valid NodeKey", client.Hostname()) + assert.Falsef(t, netmap.SelfNode.Machine().IsZero(), "%q does not have a valid MachineKey", client.Hostname()) + assert.Falsef(t, netmap.SelfNode.DiscoKey().IsZero(), "%q does not have a valid DiscoKey", client.Hostname()) + + for _, peer := range netmap.Peers { + assert.NotEqualf(t, "127.3.3.40:0", peer.DERP(), "peer (%s) has no home DERP in %q's netmap, got: %s", peer.ComputedName(), client.Hostname(), peer.DERP()) + + assert.Truef(t, peer.Hostinfo().Valid(), "peer (%s) of %q does not have Hostinfo", peer.ComputedName(), client.Hostname()) + if hi := peer.Hostinfo(); hi.Valid() { + assert.LessOrEqualf(t, 3, peer.Hostinfo().Services().Len(), "peer (%s) of %q does not have enough services, got: %v", peer.ComputedName(), client.Hostname(), peer.Hostinfo().Services()) + + // Netinfo is not always set + assert.Truef(t, hi.NetInfo().Valid(), "peer (%s) of %q does not have NetInfo", peer.ComputedName(), client.Hostname()) + if ni := hi.NetInfo(); ni.Valid() { + assert.NotEqualf(t, 0, ni.PreferredDERP(), "peer (%s) has no home DERP in %q's netmap, got: %s", peer.ComputedName(), client.Hostname(), peer.Hostinfo().NetInfo().PreferredDERP()) + } + + } + + assert.NotEmptyf(t, peer.Endpoints(), "peer (%s) of %q does not have any endpoints", peer.ComputedName(), client.Hostname()) + assert.NotEmptyf(t, peer.AllowedIPs(), "peer (%s) of %q does not have any allowed IPs", peer.ComputedName(), client.Hostname()) + assert.NotEmptyf(t, peer.Addresses(), "peer (%s) of %q does not have any addresses", peer.ComputedName(), client.Hostname()) + + assert.Truef(t, *peer.Online(), "peer (%s) of %q is not online", peer.ComputedName(), client.Hostname()) + + assert.Falsef(t, peer.Key().IsZero(), "peer (%s) of %q does not have a valid NodeKey", peer.ComputedName(), client.Hostname()) + assert.Falsef(t, peer.Machine().IsZero(), "peer (%s) of %q does not have a valid MachineKey", peer.ComputedName(), client.Hostname()) + assert.Falsef(t, peer.DiscoKey().IsZero(), "peer (%s) of %q does not have a valid DiscoKey", peer.ComputedName(), client.Hostname()) + } +} + +// assertValidStatus asserts that the status of a client has all +// the minimum required fields set to a known working config for +// the general case. Fields are checked on self, then all peers. +// This test is not suitable for ACL/partial connection tests. +func assertValidStatus(t *testing.T, client TailscaleClient) { + t.Helper() + status, err := client.Status() + if err != nil { + t.Fatalf("getting status for %q: %s", client.Hostname(), err) + } + + assert.NotEmptyf(t, status.Self.HostName, "%q does not have HostName set, likely missing Hostinfo", client.Hostname()) + assert.NotEmptyf(t, status.Self.OS, "%q does not have OS set, likely missing Hostinfo", client.Hostname()) + assert.NotEmptyf(t, status.Self.Relay, "%q does not have a relay, likely missing Hostinfo/Netinfo", client.Hostname()) + + assert.NotEmptyf(t, status.Self.TailscaleIPs, "%q does not have Tailscale IPs", client.Hostname()) + + // This seem to not appear until version 1.56 + if status.Self.AllowedIPs != nil { + assert.NotEmptyf(t, status.Self.AllowedIPs, "%q does not have any allowed IPs", client.Hostname()) + } + + assert.NotEmptyf(t, status.Self.Addrs, "%q does not have any endpoints", client.Hostname()) + + assert.Truef(t, status.Self.Online, "%q is not online", client.Hostname()) + + assert.Truef(t, status.Self.InNetworkMap, "%q is not in network map", client.Hostname()) + + // This isnt really relevant for Self as it wont be in its own socket/wireguard. + // assert.Truef(t, status.Self.InMagicSock, "%q is not tracked by magicsock", client.Hostname()) + // assert.Truef(t, status.Self.InEngine, "%q is not in in wireguard engine", client.Hostname()) + + for _, peer := range status.Peer { + assert.NotEmptyf(t, peer.HostName, "peer (%s) of %q does not have HostName set, likely missing Hostinfo", peer.DNSName, client.Hostname()) + assert.NotEmptyf(t, peer.OS, "peer (%s) of %q does not have OS set, likely missing Hostinfo", peer.DNSName, client.Hostname()) + assert.NotEmptyf(t, peer.Relay, "peer (%s) of %q does not have a relay, likely missing Hostinfo/Netinfo", peer.DNSName, client.Hostname()) + + assert.NotEmptyf(t, peer.TailscaleIPs, "peer (%s) of %q does not have Tailscale IPs", peer.DNSName, client.Hostname()) + + // This seem to not appear until version 1.56 + if peer.AllowedIPs != nil { + assert.NotEmptyf(t, peer.AllowedIPs, "peer (%s) of %q does not have any allowed IPs", peer.DNSName, client.Hostname()) + } + + // Addrs does not seem to appear in the status from peers. + // assert.NotEmptyf(t, peer.Addrs, "peer (%s) of %q does not have any endpoints", peer.DNSName, client.Hostname()) + + assert.Truef(t, peer.Online, "peer (%s) of %q is not online", peer.DNSName, client.Hostname()) + + assert.Truef(t, peer.InNetworkMap, "peer (%s) of %q is not in network map", peer.DNSName, client.Hostname()) + assert.Truef(t, peer.InMagicSock, "peer (%s) of %q is not tracked by magicsock", peer.DNSName, client.Hostname()) + + // TODO(kradalby): InEngine is only true when a proper tunnel is set up, + // there might be some interesting stuff to test here in the future. + // assert.Truef(t, peer.InEngine, "peer (%s) of %q is not in wireguard engine", peer.DNSName, client.Hostname()) + } +} + +func assertValidNetcheck(t *testing.T, client TailscaleClient) { + t.Helper() + report, err := client.Netcheck() + if err != nil { + t.Fatalf("getting status for %q: %s", client.Hostname(), err) + } + + assert.NotEqualf(t, 0, report.PreferredDERP, "%q does not have a DERP relay", client.Hostname()) +} + func isSelfClient(client TailscaleClient, addr string) bool { if addr == client.Hostname() { return true @@ -152,7 +296,7 @@ func isCI() bool { } func dockertestMaxWait() time.Duration { - wait := 60 * time.Second //nolint + wait := 120 * time.Second //nolint if isCI() { wait = 300 * time.Second //nolint