package integration import ( "net/netip" "sort" "testing" "time" "github.com/google/go-cmp/cmp" v1 "github.com/juanfont/headscale/gen/go/headscale/v1" "github.com/juanfont/headscale/hscontrol/policy" "github.com/juanfont/headscale/hscontrol/util" "github.com/juanfont/headscale/integration/hsic" "github.com/juanfont/headscale/integration/tsic" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "tailscale.com/ipn/ipnstate" "tailscale.com/net/tsaddr" "tailscale.com/types/ipproto" "tailscale.com/types/views" "tailscale.com/wgengine/filter" ) var allPorts = filter.PortRange{First: 0, Last: 0xffff} // This test is both testing the routes command and the propagation of // routes. func TestEnablingRoutes(t *testing.T) { IntegrationSkip(t) t.Parallel() user := "enable-routing" scenario, err := NewScenario(dockertestMaxWait()) require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) spec := map[string]int{ user: 3, } err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("clienableroute")) assertNoErrHeadscaleEnv(t, err) allClients, err := scenario.ListTailscaleClients() assertNoErrListClients(t, err) err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) headscale, err := scenario.Headscale() assertNoErrGetHeadscale(t, err) expectedRoutes := map[string]string{ "1": "10.0.0.0/24", "2": "10.0.1.0/24", "3": "10.0.2.0/24", } // advertise routes using the up command for _, client := range allClients { status, err := client.Status() require.NoError(t, err) command := []string{ "tailscale", "set", "--advertise-routes=" + expectedRoutes[string(status.Self.ID)], } _, _, err = client.Execute(command) require.NoErrorf(t, err, "failed to advertise route: %s", err) } err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) nodes, err := headscale.ListNodes() require.NoError(t, err) for _, node := range nodes { assert.Len(t, node.GetAvailableRoutes(), 1) assert.Empty(t, node.GetApprovedRoutes()) assert.Empty(t, node.GetSubnetRoutes()) } // Verify that no routes has been sent to the client, // they are not yet enabled. for _, client := range allClients { status, err := client.Status() require.NoError(t, err) for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) } } for _, node := range nodes { _, err := headscale.ApproveRoutes( node.GetId(), util.MustStringsToPrefixes(node.GetAvailableRoutes()), ) require.NoError(t, err) } nodes, err = headscale.ListNodes() require.NoError(t, err) for _, node := range nodes { assert.Len(t, node.GetAvailableRoutes(), 1) assert.Len(t, node.GetApprovedRoutes(), 1) assert.Len(t, node.GetSubnetRoutes(), 1) } time.Sleep(5 * time.Second) // Verify that the clients can see the new routes for _, client := range allClients { status, err := client.Status() require.NoError(t, err) for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) assert.Len(t, peerStatus.AllowedIPs.AsSlice(), 3) if peerStatus.AllowedIPs.Len() > 2 { peerRoute := peerStatus.AllowedIPs.At(2) // id starts at 1, we created routes with 0 index assert.Equalf( t, expectedRoutes[string(peerStatus.ID)], peerRoute.String(), "expected route %s to be present on peer %s (%s) in %s (%s) status", expectedRoutes[string(peerStatus.ID)], peerStatus.HostName, peerStatus.ID, client.Hostname(), client.ID(), ) } } } _, err = headscale.ApproveRoutes( 1, []netip.Prefix{netip.MustParsePrefix("10.0.1.0/24")}, ) require.NoError(t, err) _, err = headscale.ApproveRoutes( 2, []netip.Prefix{}, ) require.NoError(t, err) time.Sleep(5 * time.Second) nodes, err = headscale.ListNodes() require.NoError(t, err) for _, node := range nodes { if node.GetId() == 1 { assert.Len(t, node.GetAvailableRoutes(), 1) // 10.0.0.0/24 assert.Len(t, node.GetApprovedRoutes(), 1) // 10.0.1.0/24 assert.Empty(t, node.GetSubnetRoutes()) } else if node.GetId() == 2 { assert.Len(t, node.GetAvailableRoutes(), 1) // 10.0.1.0/24 assert.Empty(t, node.GetApprovedRoutes()) assert.Empty(t, node.GetSubnetRoutes()) } else { assert.Len(t, node.GetAvailableRoutes(), 1) // 10.0.2.0/24 assert.Len(t, node.GetApprovedRoutes(), 1) // 10.0.2.0/24 assert.Len(t, node.GetSubnetRoutes(), 1) // 10.0.2.0/24 } } // Verify that the clients can see the new routes for _, client := range allClients { status, err := client.Status() require.NoError(t, err) for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) if peerStatus.ID == "1" { assertPeerSubnetRoutes(t, peerStatus, nil) } else if peerStatus.ID == "2" { assertPeerSubnetRoutes(t, peerStatus, nil) } else { assertPeerSubnetRoutes(t, peerStatus, []netip.Prefix{netip.MustParsePrefix("10.0.2.0/24")}) } } } } func TestHASubnetRouterFailover(t *testing.T) { IntegrationSkip(t) t.Parallel() user := "enable-routing" scenario, err := NewScenario(dockertestMaxWait()) require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) spec := map[string]int{ user: 4, } err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("clienableroute"), hsic.WithEmbeddedDERPServerOnly(), hsic.WithTLS(), ) assertNoErrHeadscaleEnv(t, err) allClients, err := scenario.ListTailscaleClients() assertNoErrListClients(t, err) err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) headscale, err := scenario.Headscale() assertNoErrGetHeadscale(t, err) expectedRoutes := map[string]string{ "1": "10.0.0.0/24", "2": "10.0.0.0/24", "3": "10.0.0.0/24", } // Sort nodes by ID sort.SliceStable(allClients, func(i, j int) bool { statusI := allClients[i].MustStatus() statusJ := allClients[j].MustStatus() return statusI.Self.ID < statusJ.Self.ID }) subRouter1 := allClients[0] subRouter2 := allClients[1] subRouter3 := allClients[2] client := allClients[3] t.Logf("Advertise route from r1 (%s), r2 (%s), r3 (%s), making it HA, n1 is primary", subRouter1.Hostname(), subRouter2.Hostname(), subRouter3.Hostname()) // advertise HA route on node 1, 2, 3 // ID 1 will be primary // ID 2 will be standby // ID 3 will be standby for _, client := range allClients[:3] { status, err := client.Status() require.NoError(t, err) if route, ok := expectedRoutes[string(status.Self.ID)]; ok { command := []string{ "tailscale", "set", "--advertise-routes=" + route, } _, _, err = client.Execute(command) require.NoErrorf(t, err, "failed to advertise route: %s", err) } else { t.Fatalf("failed to find route for Node %s (id: %s)", status.Self.HostName, status.Self.ID) } } err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) nodes, err := headscale.ListNodes() require.NoError(t, err) assert.Len(t, nodes, 4) assertNodeRouteCount(t, nodes[0], 1, 0, 0) assertNodeRouteCount(t, nodes[1], 1, 0, 0) assertNodeRouteCount(t, nodes[2], 1, 0, 0) // Verify that no routes has been sent to the client, // they are not yet enabled. for _, client := range allClients { status, err := client.Status() require.NoError(t, err) for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) assertPeerSubnetRoutes(t, peerStatus, nil) } } // Enable all routes for _, node := range nodes { _, err := headscale.ApproveRoutes( node.GetId(), util.MustStringsToPrefixes(node.GetAvailableRoutes()), ) require.NoError(t, err) } nodes, err = headscale.ListNodes() require.NoError(t, err) assert.Len(t, nodes, 4) assertNodeRouteCount(t, nodes[0], 1, 1, 1) assertNodeRouteCount(t, nodes[1], 1, 1, 1) assertNodeRouteCount(t, nodes[2], 1, 1, 1) // Verify that the client has routes from the primary machine srs1 := subRouter1.MustStatus() srs2 := subRouter2.MustStatus() srs3 := subRouter3.MustStatus() clientStatus := client.MustStatus() srs1PeerStatus := clientStatus.Peer[srs1.Self.PublicKey] srs2PeerStatus := clientStatus.Peer[srs2.Self.PublicKey] srs3PeerStatus := clientStatus.Peer[srs3.Self.PublicKey] assert.True(t, srs1PeerStatus.Online, "r1 up, r2 up") assert.True(t, srs2PeerStatus.Online, "r1 up, r2 up") assert.True(t, srs3PeerStatus.Online, "r1 up, r2 up") assert.Nil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) require.NotNil(t, srs1PeerStatus.PrimaryRoutes) assert.Contains(t, srs1PeerStatus.PrimaryRoutes.AsSlice(), netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]), ) // Take down the current primary t.Logf("taking down subnet router r1 (%s)", subRouter1.Hostname()) t.Logf("expecting r2 (%s) to take over as primary", subRouter2.Hostname()) err = subRouter1.Down() require.NoError(t, err) time.Sleep(5 * time.Second) srs2 = subRouter2.MustStatus() clientStatus = client.MustStatus() srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] assert.False(t, srs1PeerStatus.Online, "r1 down, r2 down") assert.True(t, srs2PeerStatus.Online, "r1 down, r2 up") assert.True(t, srs3PeerStatus.Online, "r1 down, r2 up") assert.Nil(t, srs1PeerStatus.PrimaryRoutes) require.NotNil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) assert.Contains( t, srs2PeerStatus.PrimaryRoutes.AsSlice(), netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]), ) // Take down subnet router 2, leaving none available t.Logf("taking down subnet router r2 (%s)", subRouter2.Hostname()) t.Logf("expecting no primary, r3 available, but no HA so no primary") err = subRouter2.Down() require.NoError(t, err) time.Sleep(5 * time.Second) // TODO(kradalby): Check client status // Both are expected to be down // Verify that the route is not presented from either router clientStatus, err = client.Status() require.NoError(t, err) srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] assert.False(t, srs1PeerStatus.Online, "r1 down, r2 down") assert.False(t, srs2PeerStatus.Online, "r1 down, r2 down") assert.True(t, srs3PeerStatus.Online, "r1 down, r2 down") assert.Nil(t, srs1PeerStatus.PrimaryRoutes) assert.Nil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) // Bring up subnet router 1, making the route available from there. t.Logf("bringing up subnet router r1 (%s)", subRouter1.Hostname()) t.Logf("expecting r1 (%s) to take over as primary, r1 and r3 available", subRouter1.Hostname()) err = subRouter1.Up() require.NoError(t, err) time.Sleep(5 * time.Second) // Verify that the route is announced from subnet router 1 clientStatus, err = client.Status() require.NoError(t, err) srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] assert.True(t, srs1PeerStatus.Online, "r1 is back up, r2 down") assert.False(t, srs2PeerStatus.Online, "r1 is back up, r2 down") assert.True(t, srs3PeerStatus.Online, "r1 is back up, r3 available") assert.NotNil(t, srs1PeerStatus.PrimaryRoutes) assert.Nil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) assert.Contains( t, srs1PeerStatus.PrimaryRoutes.AsSlice(), netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]), ) // Bring up subnet router 2, should result in no change. t.Logf("bringing up subnet router r2 (%s)", subRouter2.Hostname()) t.Logf("all online, expecting r1 (%s) to still be primary (no flapping)", subRouter1.Hostname()) err = subRouter2.Up() require.NoError(t, err) time.Sleep(5 * time.Second) // Verify that the route is announced from subnet router 1 clientStatus, err = client.Status() require.NoError(t, err) srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] assert.True(t, srs1PeerStatus.Online, "r1 up, r2 up") assert.True(t, srs2PeerStatus.Online, "r1 up, r2 up") assert.True(t, srs3PeerStatus.Online, "r1 up, r2 up") require.NotNil(t, srs1PeerStatus.PrimaryRoutes) assert.Nil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) assert.Contains( t, srs1PeerStatus.PrimaryRoutes.AsSlice(), netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]), ) // Disable the route of subnet router 1, making it failover to 2 t.Logf("disabling route in subnet router r1 (%s)", subRouter1.Hostname()) t.Logf("expecting route to failover to r2 (%s), which is still available with r3", subRouter2.Hostname()) _, err = headscale.ApproveRoutes(nodes[0].GetId(), []netip.Prefix{}) time.Sleep(5 * time.Second) nodes, err = headscale.ListNodes() require.NoError(t, err) assert.Len(t, nodes, 4) assertNodeRouteCount(t, nodes[0], 1, 0, 0) assertNodeRouteCount(t, nodes[1], 1, 1, 1) assertNodeRouteCount(t, nodes[2], 1, 1, 1) // Verify that the route is announced from subnet router 1 clientStatus, err = client.Status() require.NoError(t, err) srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] assert.Nil(t, srs1PeerStatus.PrimaryRoutes) assert.NotNil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) assert.Contains( t, srs2PeerStatus.PrimaryRoutes.AsSlice(), netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]), ) // enable the route of subnet router 1, no change expected t.Logf("enabling route in subnet router 1 (%s)", subRouter1.Hostname()) t.Logf("both online, expecting r2 (%s) to still be primary (no flapping)", subRouter2.Hostname()) _, err = headscale.ApproveRoutes( nodes[0].GetId(), util.MustStringsToPrefixes(nodes[0].GetAvailableRoutes()), ) time.Sleep(5 * time.Second) nodes, err = headscale.ListNodes() require.NoError(t, err) assert.Len(t, nodes, 4) assertNodeRouteCount(t, nodes[0], 1, 1, 1) assertNodeRouteCount(t, nodes[1], 1, 1, 1) assertNodeRouteCount(t, nodes[2], 1, 1, 1) // Verify that the route is announced from subnet router 1 clientStatus, err = client.Status() require.NoError(t, err) srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] assert.Nil(t, srs1PeerStatus.PrimaryRoutes) require.NotNil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) assert.Contains( t, srs2PeerStatus.PrimaryRoutes.AsSlice(), netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]), ) } func TestEnableDisableAutoApprovedRoute(t *testing.T) { IntegrationSkip(t) t.Parallel() expectedRoutes := "172.0.0.0/24" user := "enable-disable-routing" scenario, err := NewScenario(dockertestMaxWait()) require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) spec := map[string]int{ user: 1, } err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{tsic.WithTags([]string{"tag:approve"})}, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy( &policy.ACLPolicy{ ACLs: []policy.ACL{ { Action: "accept", Sources: []string{"*"}, Destinations: []string{"*:*"}, }, }, TagOwners: map[string][]string{ "tag:approve": {user}, }, AutoApprovers: policy.AutoApprovers{ Routes: map[string][]string{ expectedRoutes: {"tag:approve"}, }, }, }, )) assertNoErrHeadscaleEnv(t, err) allClients, err := scenario.ListTailscaleClients() assertNoErrListClients(t, err) err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) headscale, err := scenario.Headscale() assertNoErrGetHeadscale(t, err) subRouter1 := allClients[0] // Initially advertise route command := []string{ "tailscale", "set", "--advertise-routes=" + expectedRoutes, } _, _, err = subRouter1.Execute(command) require.NoErrorf(t, err, "failed to advertise route: %s", err) time.Sleep(10 * time.Second) nodes, err := headscale.ListNodes() require.NoError(t, err) assert.Len(t, nodes, 1) assertNodeRouteCount(t, nodes[0], 1, 1, 1) // Stop advertising route command = []string{ "tailscale", "set", "--advertise-routes=", } _, _, err = subRouter1.Execute(command) require.NoErrorf(t, err, "failed to remove advertised route: %s", err) time.Sleep(10 * time.Second) nodes, err = headscale.ListNodes() require.NoError(t, err) assert.Len(t, nodes, 1) assertNodeRouteCount(t, nodes[0], 0, 1, 0) // Advertise route again command = []string{ "tailscale", "set", "--advertise-routes=" + expectedRoutes, } _, _, err = subRouter1.Execute(command) require.NoErrorf(t, err, "failed to advertise route: %s", err) time.Sleep(10 * time.Second) nodes, err = headscale.ListNodes() require.NoError(t, err) assert.Len(t, nodes, 1) assertNodeRouteCount(t, nodes[0], 1, 1, 1) } func TestAutoApprovedSubRoute2068(t *testing.T) { IntegrationSkip(t) t.Parallel() expectedRoutes := "10.42.7.0/24" user := "user1" scenario, err := NewScenario(dockertestMaxWait()) require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) spec := map[string]int{ user: 1, } err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{tsic.WithTags([]string{"tag:approve"})}, hsic.WithTestName("clienableroute"), hsic.WithEmbeddedDERPServerOnly(), hsic.WithTLS(), hsic.WithACLPolicy( &policy.ACLPolicy{ ACLs: []policy.ACL{ { Action: "accept", Sources: []string{"*"}, Destinations: []string{"*:*"}, }, }, TagOwners: map[string][]string{ "tag:approve": {user}, }, AutoApprovers: policy.AutoApprovers{ Routes: map[string][]string{ "10.42.0.0/16": {"tag:approve"}, }, }, }, )) assertNoErrHeadscaleEnv(t, err) allClients, err := scenario.ListTailscaleClients() assertNoErrListClients(t, err) err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) headscale, err := scenario.Headscale() assertNoErrGetHeadscale(t, err) subRouter1 := allClients[0] // Initially advertise route command := []string{ "tailscale", "set", "--advertise-routes=" + expectedRoutes, } _, _, err = subRouter1.Execute(command) require.NoErrorf(t, err, "failed to advertise route: %s", err) time.Sleep(10 * time.Second) nodes, err := headscale.ListNodes() require.NoError(t, err) assert.Len(t, nodes, 1) assertNodeRouteCount(t, nodes[0], 1, 1, 1) } // TestSubnetRouteACL verifies that Subnet routes are distributed // as expected when ACLs are activated. // It implements the issue from // https://github.com/juanfont/headscale/issues/1604 func TestSubnetRouteACL(t *testing.T) { IntegrationSkip(t) t.Parallel() user := "subnet-route-acl" scenario, err := NewScenario(dockertestMaxWait()) require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) spec := map[string]int{ user: 2, } err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy( &policy.ACLPolicy{ Groups: policy.Groups{ "group:admins": {user}, }, ACLs: []policy.ACL{ { Action: "accept", Sources: []string{"group:admins"}, Destinations: []string{"group:admins:*"}, }, { Action: "accept", Sources: []string{"group:admins"}, Destinations: []string{"10.33.0.0/16:*"}, }, // { // Action: "accept", // Sources: []string{"group:admins"}, // Destinations: []string{"0.0.0.0/0:*"}, // }, }, }, )) assertNoErrHeadscaleEnv(t, err) allClients, err := scenario.ListTailscaleClients() assertNoErrListClients(t, err) err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) headscale, err := scenario.Headscale() assertNoErrGetHeadscale(t, err) expectedRoutes := map[string]string{ "1": "10.33.0.0/16", } // Sort nodes by ID sort.SliceStable(allClients, func(i, j int) bool { statusI, err := allClients[i].Status() if err != nil { return false } statusJ, err := allClients[j].Status() if err != nil { return false } return statusI.Self.ID < statusJ.Self.ID }) subRouter1 := allClients[0] client := allClients[1] for _, client := range allClients { status, err := client.Status() require.NoError(t, err) if route, ok := expectedRoutes[string(status.Self.ID)]; ok { command := []string{ "tailscale", "set", "--advertise-routes=" + route, } _, _, err = client.Execute(command) require.NoErrorf(t, err, "failed to advertise route: %s", err) } } err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) nodes, err := headscale.ListNodes() require.NoError(t, err) require.Len(t, nodes, 2) assertNodeRouteCount(t, nodes[0], 1, 0, 0) assertNodeRouteCount(t, nodes[1], 0, 0, 0) // Verify that no routes has been sent to the client, // they are not yet enabled. for _, client := range allClients { status, err := client.Status() require.NoError(t, err) for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) assertPeerSubnetRoutes(t, peerStatus, nil) } } _, err = headscale.ApproveRoutes( 1, []netip.Prefix{netip.MustParsePrefix(expectedRoutes["1"])}, ) require.NoError(t, err) time.Sleep(5 * time.Second) nodes, err = headscale.ListNodes() require.NoError(t, err) require.Len(t, nodes, 2) assertNodeRouteCount(t, nodes[0], 1, 1, 1) assertNodeRouteCount(t, nodes[1], 0, 0, 0) // Verify that the client has routes from the primary machine srs1, _ := subRouter1.Status() clientStatus, err := client.Status() require.NoError(t, err) srs1PeerStatus := clientStatus.Peer[srs1.Self.PublicKey] assertPeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{netip.MustParsePrefix(expectedRoutes["1"])}) clientNm, err := client.Netmap() require.NoError(t, err) wantClientFilter := []filter.Match{ { IPProto: views.SliceOf([]ipproto.Proto{ ipproto.TCP, ipproto.UDP, ipproto.ICMPv4, ipproto.ICMPv6, }), Srcs: []netip.Prefix{ netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("100.64.0.2/32"), netip.MustParsePrefix("fd7a:115c:a1e0::1/128"), netip.MustParsePrefix("fd7a:115c:a1e0::2/128"), }, Dsts: []filter.NetPortRange{ { Net: netip.MustParsePrefix("100.64.0.2/32"), Ports: allPorts, }, { Net: netip.MustParsePrefix("fd7a:115c:a1e0::2/128"), Ports: allPorts, }, }, Caps: []filter.CapMatch{}, }, } if diff := cmp.Diff(wantClientFilter, clientNm.PacketFilter, util.ViewSliceIPProtoComparer, util.PrefixComparer); diff != "" { t.Errorf("Client (%s) filter, unexpected result (-want +got):\n%s", client.Hostname(), diff) } subnetNm, err := subRouter1.Netmap() require.NoError(t, err) wantSubnetFilter := []filter.Match{ { IPProto: views.SliceOf([]ipproto.Proto{ ipproto.TCP, ipproto.UDP, ipproto.ICMPv4, ipproto.ICMPv6, }), Srcs: []netip.Prefix{ netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("100.64.0.2/32"), netip.MustParsePrefix("fd7a:115c:a1e0::1/128"), netip.MustParsePrefix("fd7a:115c:a1e0::2/128"), }, Dsts: []filter.NetPortRange{ { Net: netip.MustParsePrefix("100.64.0.1/32"), Ports: allPorts, }, { Net: netip.MustParsePrefix("fd7a:115c:a1e0::1/128"), Ports: allPorts, }, }, Caps: []filter.CapMatch{}, }, { IPProto: views.SliceOf([]ipproto.Proto{ ipproto.TCP, ipproto.UDP, ipproto.ICMPv4, ipproto.ICMPv6, }), Srcs: []netip.Prefix{ netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("100.64.0.2/32"), netip.MustParsePrefix("fd7a:115c:a1e0::1/128"), netip.MustParsePrefix("fd7a:115c:a1e0::2/128"), }, Dsts: []filter.NetPortRange{ { Net: netip.MustParsePrefix("10.33.0.0/16"), Ports: allPorts, }, }, Caps: []filter.CapMatch{}, }, } if diff := cmp.Diff(wantSubnetFilter, subnetNm.PacketFilter, util.ViewSliceIPProtoComparer, util.PrefixComparer); diff != "" { t.Errorf("Subnet (%s) filter, unexpected result (-want +got):\n%s", subRouter1.Hostname(), diff) } } // TestEnablingExitRoutes tests enabling exit routes for clients. // Its more or less the same as TestEnablingRoutes, but with the --advertise-exit-node flag // set during login instead of set. func TestEnablingExitRoutes(t *testing.T) { IntegrationSkip(t) t.Parallel() user := "user2" scenario, err := NewScenario(dockertestMaxWait()) assertNoErrf(t, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) spec := map[string]int{ user: 2, } err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{ tsic.WithExtraLoginArgs([]string{"--advertise-exit-node"}), }, hsic.WithTestName("clienableroute")) assertNoErrHeadscaleEnv(t, err) allClients, err := scenario.ListTailscaleClients() assertNoErrListClients(t, err) err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) headscale, err := scenario.Headscale() assertNoErrGetHeadscale(t, err) err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) nodes, err := headscale.ListNodes() require.NoError(t, err) require.Len(t, nodes, 2) assertNodeRouteCount(t, nodes[0], 2, 0, 0) assertNodeRouteCount(t, nodes[1], 2, 0, 0) // Verify that no routes has been sent to the client, // they are not yet enabled. for _, client := range allClients { status, err := client.Status() assertNoErr(t, err) for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) } } // Enable all routes, but do v4 on one and v6 on other to ensure they // are both added since they are exit routes. _, err = headscale.ApproveRoutes( nodes[0].GetId(), []netip.Prefix{tsaddr.AllIPv4()}, ) require.NoError(t, err) _, err = headscale.ApproveRoutes( nodes[1].GetId(), []netip.Prefix{tsaddr.AllIPv6()}, ) require.NoError(t, err) nodes, err = headscale.ListNodes() require.NoError(t, err) require.Len(t, nodes, 2) assertNodeRouteCount(t, nodes[0], 2, 2, 2) assertNodeRouteCount(t, nodes[1], 2, 2, 2) time.Sleep(5 * time.Second) // Verify that the clients can see the new routes for _, client := range allClients { status, err := client.Status() assertNoErr(t, err) for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] require.NotNil(t, peerStatus.AllowedIPs) assert.Len(t, peerStatus.AllowedIPs.AsSlice(), 4) assert.Contains(t, peerStatus.AllowedIPs.AsSlice(), tsaddr.AllIPv4()) assert.Contains(t, peerStatus.AllowedIPs.AsSlice(), tsaddr.AllIPv6()) } } } // assertPeerSubnetRoutes asserts that the peer has the expected subnet routes. func assertPeerSubnetRoutes(t *testing.T, status *ipnstate.PeerStatus, expected []netip.Prefix) { t.Helper() if status.AllowedIPs.Len() <= 2 && len(expected) != 0 { t.Errorf("peer %s (%s) has no subnet routes, expected %v", status.HostName, status.ID, expected) return } if len(expected) == 0 { expected = []netip.Prefix{} } got := status.AllowedIPs.AsSlice()[2:] if diff := cmp.Diff(expected, got, util.PrefixComparer); diff != "" { t.Errorf("peer %s (%s) subnet routes, unexpected result (-want +got):\n%s", status.HostName, status.ID, diff) } } func assertNodeRouteCount(t *testing.T, node *v1.Node, announced, approved, subnet int) { t.Helper() assert.Len(t, node.GetAvailableRoutes(), announced) assert.Len(t, node.GetApprovedRoutes(), approved) assert.Len(t, node.GetSubnetRoutes(), subnet) }