hscontrol/oidc: fix ACL policy not applied to new OIDC nodes (#2890)

Fixes #2888
Fixes #2896
This commit is contained in:
Kristoffer Dalby
2025-11-30 19:02:15 +01:00
committed by GitHub
parent 7e8cee6b10
commit 4fe5cbe703
9 changed files with 757 additions and 107 deletions

View File

@@ -9,12 +9,15 @@ import (
"time"
v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/integration/hsic"
"github.com/juanfont/headscale/integration/tsic"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/tailcfg"
"tailscale.com/types/ptr"
)
func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
@@ -540,3 +543,182 @@ func TestAuthKeyDeleteKey(t *testing.T) {
t.Logf("✓ Node successfully reconnected after its auth key was deleted")
}
// TestAuthKeyLogoutAndReloginRoutesPreserved tests that routes remain serving
// after a node logs out and re-authenticates with the same user.
//
// This test validates the fix for issue #2896:
// https://github.com/juanfont/headscale/issues/2896
//
// Bug: When a node with already-approved routes restarts/re-authenticates,
// the routes show as "Approved" and "Available" but NOT "Serving" (Primary).
// A headscale restart would fix it, indicating a state management issue.
//
// The test scenario:
// 1. Node registers with auth key and advertises routes
// 2. Routes are auto-approved and verified as serving
// 3. Node logs out
// 4. Node re-authenticates with same auth key
// 5. Routes should STILL be serving (this is where the bug manifests)
func TestAuthKeyLogoutAndReloginRoutesPreserved(t *testing.T) {
IntegrationSkip(t)
user := "routeuser"
advertiseRoute := "10.55.0.0/24"
spec := ScenarioSpec{
NodesPerUser: 1,
Users: []string{user},
}
scenario, err := NewScenario(spec)
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
err = scenario.CreateHeadscaleEnv(
[]tsic.Option{
tsic.WithAcceptRoutes(),
// Advertise route on initial login
tsic.WithExtraLoginArgs([]string{"--advertise-routes=" + advertiseRoute}),
},
hsic.WithTestName("routelogout"),
hsic.WithTLS(),
hsic.WithACLPolicy(
&policyv2.Policy{
ACLs: []policyv2.ACL{
{
Action: "accept",
Sources: []policyv2.Alias{policyv2.Wildcard},
Destinations: []policyv2.AliasWithPorts{{Alias: policyv2.Wildcard, Ports: []tailcfg.PortRange{tailcfg.PortRangeAny}}},
},
},
AutoApprovers: policyv2.AutoApproverPolicy{
Routes: map[netip.Prefix]policyv2.AutoApprovers{
netip.MustParsePrefix(advertiseRoute): {ptr.To(policyv2.Username(user + "@test.no"))},
},
},
},
),
)
requireNoErrHeadscaleEnv(t, err)
allClients, err := scenario.ListTailscaleClients()
requireNoErrListClients(t, err)
require.Len(t, allClients, 1)
client := allClients[0]
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
headscale, err := scenario.Headscale()
requireNoErrGetHeadscale(t, err)
// Step 1: Verify initial route is advertised, approved, and SERVING
t.Logf("Step 1: Verifying initial route is advertised, approved, and SERVING at %s", time.Now().Format(TimestampFormat))
var initialNode *v1.Node
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, nodes, 1, "Should have exactly 1 node")
if len(nodes) == 1 {
initialNode = nodes[0]
// Check: 1 announced, 1 approved, 1 serving (subnet route)
assert.Lenf(c, initialNode.GetAvailableRoutes(), 1,
"Node should have 1 available route, got %v", initialNode.GetAvailableRoutes())
assert.Lenf(c, initialNode.GetApprovedRoutes(), 1,
"Node should have 1 approved route, got %v", initialNode.GetApprovedRoutes())
assert.Lenf(c, initialNode.GetSubnetRoutes(), 1,
"Node should have 1 serving (subnet) route, got %v - THIS IS THE BUG if empty", initialNode.GetSubnetRoutes())
assert.Contains(c, initialNode.GetSubnetRoutes(), advertiseRoute,
"Subnet routes should contain %s", advertiseRoute)
}
}, 30*time.Second, 500*time.Millisecond, "initial route should be serving")
require.NotNil(t, initialNode, "Initial node should be found")
initialNodeID := initialNode.GetId()
t.Logf("Initial node ID: %d, Available: %v, Approved: %v, Serving: %v",
initialNodeID, initialNode.GetAvailableRoutes(), initialNode.GetApprovedRoutes(), initialNode.GetSubnetRoutes())
// Step 2: Logout
t.Logf("Step 2: Logging out at %s", time.Now().Format(TimestampFormat))
err = client.Logout()
require.NoError(t, err)
// Wait for logout to complete
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
status, err := client.Status()
assert.NoError(ct, err)
assert.Equal(ct, "NeedsLogin", status.BackendState, "Expected NeedsLogin state after logout")
}, 30*time.Second, 1*time.Second, "waiting for logout to complete")
t.Logf("Logout completed, node should still exist in database")
// Verify node still exists (routes should still be in DB)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, nodes, 1, "Node should persist in database after logout")
}, 10*time.Second, 500*time.Millisecond, "node should persist after logout")
// Step 3: Re-authenticate with the SAME user (using auth key)
t.Logf("Step 3: Re-authenticating with same user at %s", time.Now().Format(TimestampFormat))
userMap, err := headscale.MapUsers()
require.NoError(t, err)
key, err := scenario.CreatePreAuthKey(userMap[user].GetId(), true, false)
require.NoError(t, err)
// Re-login - the container already has extraLoginArgs with --advertise-routes
// from the initial setup, so routes will be advertised on re-login
err = scenario.RunTailscaleUp(user, headscale.GetEndpoint(), key.GetKey())
require.NoError(t, err)
// Wait for client to be running
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
status, err := client.Status()
assert.NoError(ct, err)
assert.Equal(ct, "Running", status.BackendState, "Expected Running state after relogin")
}, 30*time.Second, 1*time.Second, "waiting for relogin to complete")
t.Logf("Re-authentication completed at %s", time.Now().Format(TimestampFormat))
// Step 4: THE CRITICAL TEST - Verify routes are STILL SERVING after re-authentication
t.Logf("Step 4: Verifying routes are STILL SERVING after re-authentication at %s", time.Now().Format(TimestampFormat))
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, nodes, 1, "Should still have exactly 1 node after relogin")
if len(nodes) == 1 {
node := nodes[0]
t.Logf("After relogin - Available: %v, Approved: %v, Serving: %v",
node.GetAvailableRoutes(), node.GetApprovedRoutes(), node.GetSubnetRoutes())
// This is where issue #2896 manifests:
// - Available shows the route (from Hostinfo.RoutableIPs)
// - Approved shows the route (from ApprovedRoutes)
// - BUT Serving (SubnetRoutes/PrimaryRoutes) is EMPTY!
assert.Lenf(c, node.GetAvailableRoutes(), 1,
"Node should have 1 available route after relogin, got %v", node.GetAvailableRoutes())
assert.Lenf(c, node.GetApprovedRoutes(), 1,
"Node should have 1 approved route after relogin, got %v", node.GetApprovedRoutes())
assert.Lenf(c, node.GetSubnetRoutes(), 1,
"BUG #2896: Node should have 1 SERVING route after relogin, got %v", node.GetSubnetRoutes())
assert.Contains(c, node.GetSubnetRoutes(), advertiseRoute,
"BUG #2896: Subnet routes should contain %s after relogin", advertiseRoute)
// Also verify node ID was preserved (same node, not new registration)
assert.Equal(c, initialNodeID, node.GetId(),
"Node ID should be preserved after same-user relogin")
}
}, 30*time.Second, 500*time.Millisecond,
"BUG #2896: routes should remain SERVING after logout/relogin with same user")
t.Logf("Test completed - verifying issue #2896 fix")
}

View File

@@ -12,6 +12,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/integration/hsic"
"github.com/juanfont/headscale/integration/tsic"
@@ -19,6 +20,8 @@ import (
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/ipn/ipnstate"
"tailscale.com/tailcfg"
)
func TestOIDCAuthenticationPingAll(t *testing.T) {
@@ -1422,3 +1425,490 @@ func TestOIDCExpiryAfterRestart(t *testing.T) {
}
}, 30*time.Second, 1*time.Second, "validating expiry preservation after restart")
}
// TestOIDCACLPolicyOnJoin validates that ACL policies are correctly applied
// to newly joined OIDC nodes without requiring a client restart.
//
// This test validates the fix for issue #2888:
// https://github.com/juanfont/headscale/issues/2888
//
// Bug: Nodes joining via OIDC authentication did not get the appropriate ACL
// policy applied until they restarted their client. This was a regression
// introduced in v0.27.0.
//
// The test scenario:
// 1. Creates a CLI user (gateway) with a node advertising a route
// 2. Sets up ACL policy allowing all nodes to access advertised routes
// 3. OIDC user authenticates and joins with a new node
// 4. Verifies that the OIDC user's node IMMEDIATELY sees the advertised route
//
// Expected behavior:
// - Without fix: OIDC node cannot see the route (PrimaryRoutes is nil/empty)
// - With fix: OIDC node immediately sees the route in PrimaryRoutes
//
// Root cause: The buggy code called a.h.Change(c) immediately after user
// creation but BEFORE node registration completed, creating a race condition
// where policy change notifications were sent asynchronously before the node
// was fully registered.
func TestOIDCACLPolicyOnJoin(t *testing.T) {
IntegrationSkip(t)
gatewayUser := "gateway"
oidcUser := "oidcuser"
spec := ScenarioSpec{
NodesPerUser: 1,
Users: []string{gatewayUser},
OIDCUsers: []mockoidc.MockUser{
oidcMockUser(oidcUser, true),
},
}
scenario, err := NewScenario(spec)
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
oidcMap := map[string]string{
"HEADSCALE_OIDC_ISSUER": scenario.mockOIDC.Issuer(),
"HEADSCALE_OIDC_CLIENT_ID": scenario.mockOIDC.ClientID(),
"CREDENTIALS_DIRECTORY_TEST": "/tmp",
"HEADSCALE_OIDC_CLIENT_SECRET_PATH": "${CREDENTIALS_DIRECTORY_TEST}/hs_client_oidc_secret",
}
// Create headscale environment with ACL policy that allows OIDC user
// to access routes advertised by gateway user
err = scenario.CreateHeadscaleEnvWithLoginURL(
[]tsic.Option{
tsic.WithAcceptRoutes(),
},
hsic.WithTestName("oidcaclpolicy"),
hsic.WithConfigEnv(oidcMap),
hsic.WithTLS(),
hsic.WithFileInContainer("/tmp/hs_client_oidc_secret", []byte(scenario.mockOIDC.ClientSecret())),
hsic.WithACLPolicy(
&policyv2.Policy{
ACLs: []policyv2.ACL{
{
Action: "accept",
Sources: []policyv2.Alias{prefixp("100.64.0.0/10")},
Destinations: []policyv2.AliasWithPorts{
aliasWithPorts(prefixp("100.64.0.0/10"), tailcfg.PortRangeAny),
aliasWithPorts(prefixp("10.33.0.0/24"), tailcfg.PortRangeAny),
aliasWithPorts(prefixp("10.44.0.0/24"), tailcfg.PortRangeAny),
},
},
},
AutoApprovers: policyv2.AutoApproverPolicy{
Routes: map[netip.Prefix]policyv2.AutoApprovers{
netip.MustParsePrefix("10.33.0.0/24"): {usernameApprover("gateway@test.no"), usernameApprover("oidcuser@headscale.net"), usernameApprover("jane.doe@example.com")},
netip.MustParsePrefix("10.44.0.0/24"): {usernameApprover("gateway@test.no"), usernameApprover("oidcuser@headscale.net"), usernameApprover("jane.doe@example.com")},
},
},
},
),
)
requireNoErrHeadscaleEnv(t, err)
headscale, err := scenario.Headscale()
require.NoError(t, err)
// Get the gateway client (CLI user) - only one client at first
allClients, err := scenario.ListTailscaleClients()
requireNoErrListClients(t, err)
require.Len(t, allClients, 1, "Should have exactly 1 client (gateway) before OIDC login")
gatewayClient := allClients[0]
// Wait for initial sync (gateway logs in)
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
// Gateway advertises route 10.33.0.0/24
advertiseRoute := "10.33.0.0/24"
command := []string{
"tailscale",
"set",
"--advertise-routes=" + advertiseRoute,
}
_, _, err = gatewayClient.Execute(command)
require.NoErrorf(t, err, "failed to advertise route: %s", err)
// Wait for route advertisement to propagate
var gatewayNodeID uint64
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(ct, err)
assert.Len(ct, nodes, 1)
gatewayNode := nodes[0]
gatewayNodeID = gatewayNode.GetId()
assert.Len(ct, gatewayNode.GetAvailableRoutes(), 1)
assert.Contains(ct, gatewayNode.GetAvailableRoutes(), advertiseRoute)
}, 10*time.Second, 500*time.Millisecond, "route advertisement should propagate to headscale")
// Approve the advertised route
_, err = headscale.ApproveRoutes(
gatewayNodeID,
[]netip.Prefix{netip.MustParsePrefix(advertiseRoute)},
)
require.NoError(t, err)
// Wait for route approval to propagate
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(ct, err)
assert.Len(ct, nodes, 1)
gatewayNode := nodes[0]
assert.Len(ct, gatewayNode.GetApprovedRoutes(), 1)
assert.Contains(ct, gatewayNode.GetApprovedRoutes(), advertiseRoute)
}, 10*time.Second, 500*time.Millisecond, "route approval should propagate to headscale")
// NOW create the OIDC user by having them join
// This is where issue #2888 manifests - the new OIDC node should immediately
// see the gateway's advertised route
t.Logf("OIDC user joining at %s", time.Now().Format(TimestampFormat))
// Create OIDC user's tailscale node
oidcAdvertiseRoute := "10.44.0.0/24"
oidcClient, err := scenario.CreateTailscaleNode(
"head",
tsic.WithNetwork(scenario.networks[scenario.testDefaultNetwork]),
tsic.WithAcceptRoutes(),
tsic.WithExtraLoginArgs([]string{"--advertise-routes=" + oidcAdvertiseRoute}),
)
require.NoError(t, err)
// OIDC login happens automatically via LoginWithURL
loginURL, err := oidcClient.LoginWithURL(headscale.GetEndpoint())
require.NoError(t, err)
_, err = doLoginURL(oidcClient.Hostname(), loginURL)
require.NoError(t, err)
t.Logf("OIDC user logged in successfully at %s", time.Now().Format(TimestampFormat))
// THE CRITICAL TEST: Verify that the OIDC user's node can IMMEDIATELY
// see the gateway's advertised route WITHOUT needing a client restart.
//
// This is where the bug manifests:
// - Without fix: PrimaryRoutes will be nil/empty
// - With fix: PrimaryRoutes immediately contains the advertised route
t.Logf("Verifying OIDC user can immediately see advertised routes at %s", time.Now().Format(TimestampFormat))
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
status, err := oidcClient.Status()
assert.NoError(ct, err)
// Find the gateway peer in the OIDC user's peer list
var gatewayPeer *ipnstate.PeerStatus
for _, peerKey := range status.Peers() {
peer := status.Peer[peerKey]
// Gateway is the peer that's not the OIDC user
if peer.UserID != status.Self.UserID {
gatewayPeer = peer
break
}
}
assert.NotNil(ct, gatewayPeer, "OIDC user should see gateway as peer")
if gatewayPeer != nil {
// This is the critical assertion - PrimaryRoutes should NOT be nil
assert.NotNil(ct, gatewayPeer.PrimaryRoutes,
"BUG #2888: Gateway peer PrimaryRoutes is nil - ACL policy not applied to new OIDC node!")
if gatewayPeer.PrimaryRoutes != nil {
routes := gatewayPeer.PrimaryRoutes.AsSlice()
assert.Contains(ct, routes, netip.MustParsePrefix(advertiseRoute),
"OIDC user should immediately see gateway's advertised route %s in PrimaryRoutes", advertiseRoute)
t.Logf("SUCCESS: OIDC user can see advertised route %s in gateway's PrimaryRoutes", advertiseRoute)
}
// Also verify AllowedIPs includes the route
if gatewayPeer.AllowedIPs != nil && gatewayPeer.AllowedIPs.Len() > 0 {
allowedIPs := gatewayPeer.AllowedIPs.AsSlice()
t.Logf("Gateway peer AllowedIPs: %v", allowedIPs)
}
}
}, 15*time.Second, 500*time.Millisecond,
"OIDC user should immediately see gateway's advertised route without client restart (issue #2888)")
// Verify that the Gateway node sees the OIDC node's advertised route (AutoApproveRoutes check)
t.Logf("Verifying Gateway user can immediately see OIDC advertised routes at %s", time.Now().Format(TimestampFormat))
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
status, err := gatewayClient.Status()
assert.NoError(ct, err)
// Find the OIDC peer in the Gateway user's peer list
var oidcPeer *ipnstate.PeerStatus
for _, peerKey := range status.Peers() {
peer := status.Peer[peerKey]
if peer.UserID != status.Self.UserID {
oidcPeer = peer
break
}
}
assert.NotNil(ct, oidcPeer, "Gateway user should see OIDC user as peer")
if oidcPeer != nil {
assert.NotNil(ct, oidcPeer.PrimaryRoutes,
"BUG: OIDC peer PrimaryRoutes is nil - AutoApproveRoutes failed or overwritten!")
if oidcPeer.PrimaryRoutes != nil {
routes := oidcPeer.PrimaryRoutes.AsSlice()
assert.Contains(ct, routes, netip.MustParsePrefix(oidcAdvertiseRoute),
"Gateway user should immediately see OIDC's advertised route %s in PrimaryRoutes", oidcAdvertiseRoute)
}
}
}, 15*time.Second, 500*time.Millisecond,
"Gateway user should immediately see OIDC's advertised route (AutoApproveRoutes check)")
// Additional validation: Verify nodes in headscale match expectations
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(ct, err)
assert.Len(ct, nodes, 2, "Should have 2 nodes (gateway + oidcuser)")
// Verify OIDC user was created correctly
users, err := headscale.ListUsers()
assert.NoError(ct, err)
// Note: mockoidc may create additional default users (like jane.doe)
// so we check for at least 2 users, not exactly 2
assert.GreaterOrEqual(ct, len(users), 2, "Should have at least 2 users (gateway CLI user + oidcuser)")
// Find gateway CLI user
var gatewayUser *v1.User
for _, user := range users {
if user.GetName() == "gateway" && user.GetProvider() == "" {
gatewayUser = user
break
}
}
assert.NotNil(ct, gatewayUser, "Should have gateway CLI user")
if gatewayUser != nil {
assert.Equal(ct, "gateway", gatewayUser.GetName())
}
// Find OIDC user
var oidcUserFound *v1.User
for _, user := range users {
if user.GetName() == "oidcuser" && user.GetProvider() == "oidc" {
oidcUserFound = user
break
}
}
assert.NotNil(ct, oidcUserFound, "Should have OIDC user")
if oidcUserFound != nil {
assert.Equal(ct, "oidcuser", oidcUserFound.GetName())
assert.Equal(ct, "oidcuser@headscale.net", oidcUserFound.GetEmail())
}
}, 10*time.Second, 500*time.Millisecond, "headscale should have correct users and nodes")
t.Logf("Test completed successfully - issue #2888 fix validated")
}
// TestOIDCReloginSameUserRoutesPreserved tests the scenario where:
// - A node logs in via OIDC and advertises routes
// - Routes are auto-approved and verified as SERVING
// - The node logs out
// - The node logs back in as the same user
// - Routes should STILL be SERVING (not just approved/available)
//
// This test validates the fix for issue #2896:
// https://github.com/juanfont/headscale/issues/2896
//
// Bug: When a node with already-approved routes restarts/re-authenticates,
// the routes show as "Approved" and "Available" but NOT "Serving" (Primary).
// A headscale restart would fix it, indicating a state management issue.
func TestOIDCReloginSameUserRoutesPreserved(t *testing.T) {
IntegrationSkip(t)
advertiseRoute := "10.55.0.0/24"
// Create scenario with same user for both login attempts
scenario, err := NewScenario(ScenarioSpec{
OIDCUsers: []mockoidc.MockUser{
oidcMockUser("user1", true), // Initial login
oidcMockUser("user1", true), // Relogin with same user
},
})
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
oidcMap := map[string]string{
"HEADSCALE_OIDC_ISSUER": scenario.mockOIDC.Issuer(),
"HEADSCALE_OIDC_CLIENT_ID": scenario.mockOIDC.ClientID(),
"CREDENTIALS_DIRECTORY_TEST": "/tmp",
"HEADSCALE_OIDC_CLIENT_SECRET_PATH": "${CREDENTIALS_DIRECTORY_TEST}/hs_client_oidc_secret",
}
err = scenario.CreateHeadscaleEnvWithLoginURL(
[]tsic.Option{
tsic.WithAcceptRoutes(),
},
hsic.WithTestName("oidcrouterelogin"),
hsic.WithConfigEnv(oidcMap),
hsic.WithTLS(),
hsic.WithFileInContainer("/tmp/hs_client_oidc_secret", []byte(scenario.mockOIDC.ClientSecret())),
hsic.WithEmbeddedDERPServerOnly(),
hsic.WithDERPAsIP(),
hsic.WithACLPolicy(
&policyv2.Policy{
ACLs: []policyv2.ACL{
{
Action: "accept",
Sources: []policyv2.Alias{policyv2.Wildcard},
Destinations: []policyv2.AliasWithPorts{{Alias: policyv2.Wildcard, Ports: []tailcfg.PortRange{tailcfg.PortRangeAny}}},
},
},
AutoApprovers: policyv2.AutoApproverPolicy{
Routes: map[netip.Prefix]policyv2.AutoApprovers{
netip.MustParsePrefix(advertiseRoute): {usernameApprover("user1@headscale.net")},
},
},
},
),
)
requireNoErrHeadscaleEnv(t, err)
headscale, err := scenario.Headscale()
require.NoError(t, err)
// Create client with route advertisement
ts, err := scenario.CreateTailscaleNode(
"unstable",
tsic.WithNetwork(scenario.networks[scenario.testDefaultNetwork]),
tsic.WithAcceptRoutes(),
tsic.WithExtraLoginArgs([]string{"--advertise-routes=" + advertiseRoute}),
)
require.NoError(t, err)
// Initial login as user1
u, err := ts.LoginWithURL(headscale.GetEndpoint())
require.NoError(t, err)
_, err = doLoginURL(ts.Hostname(), u)
require.NoError(t, err)
// Wait for client to be running
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
status, err := ts.Status()
assert.NoError(ct, err)
assert.Equal(ct, "Running", status.BackendState)
}, 30*time.Second, 1*time.Second, "waiting for initial login to complete")
// Step 1: Verify initial route is advertised, approved, and SERVING
t.Logf("Step 1: Verifying initial route is advertised, approved, and SERVING at %s", time.Now().Format(TimestampFormat))
var initialNode *v1.Node
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, nodes, 1, "Should have exactly 1 node")
if len(nodes) == 1 {
initialNode = nodes[0]
// Check: 1 announced, 1 approved, 1 serving (subnet route)
assert.Lenf(c, initialNode.GetAvailableRoutes(), 1,
"Node should have 1 available route, got %v", initialNode.GetAvailableRoutes())
assert.Lenf(c, initialNode.GetApprovedRoutes(), 1,
"Node should have 1 approved route, got %v", initialNode.GetApprovedRoutes())
assert.Lenf(c, initialNode.GetSubnetRoutes(), 1,
"Node should have 1 serving (subnet) route, got %v - THIS IS THE BUG if empty", initialNode.GetSubnetRoutes())
assert.Contains(c, initialNode.GetSubnetRoutes(), advertiseRoute,
"Subnet routes should contain %s", advertiseRoute)
}
}, 30*time.Second, 500*time.Millisecond, "initial route should be serving")
require.NotNil(t, initialNode, "Initial node should be found")
initialNodeID := initialNode.GetId()
t.Logf("Initial node ID: %d, Available: %v, Approved: %v, Serving: %v",
initialNodeID, initialNode.GetAvailableRoutes(), initialNode.GetApprovedRoutes(), initialNode.GetSubnetRoutes())
// Step 2: Logout
t.Logf("Step 2: Logging out at %s", time.Now().Format(TimestampFormat))
err = ts.Logout()
require.NoError(t, err)
// Wait for logout to complete
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
status, err := ts.Status()
assert.NoError(ct, err)
assert.Equal(ct, "NeedsLogin", status.BackendState, "Expected NeedsLogin state after logout")
}, 30*time.Second, 1*time.Second, "waiting for logout to complete")
t.Logf("Logout completed, node should still exist in database")
// Verify node still exists (routes should still be in DB)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, nodes, 1, "Node should persist in database after logout")
}, 10*time.Second, 500*time.Millisecond, "node should persist after logout")
// Step 3: Re-authenticate via OIDC as the same user
t.Logf("Step 3: Re-authenticating with same user via OIDC at %s", time.Now().Format(TimestampFormat))
u, err = ts.LoginWithURL(headscale.GetEndpoint())
require.NoError(t, err)
_, err = doLoginURL(ts.Hostname(), u)
require.NoError(t, err)
// Wait for client to be running
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
status, err := ts.Status()
assert.NoError(ct, err)
assert.Equal(ct, "Running", status.BackendState, "Expected Running state after relogin")
}, 30*time.Second, 1*time.Second, "waiting for relogin to complete")
t.Logf("Re-authentication completed at %s", time.Now().Format(TimestampFormat))
// Step 4: THE CRITICAL TEST - Verify routes are STILL SERVING after re-authentication
t.Logf("Step 4: Verifying routes are STILL SERVING after re-authentication at %s", time.Now().Format(TimestampFormat))
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, nodes, 1, "Should still have exactly 1 node after relogin")
if len(nodes) == 1 {
node := nodes[0]
t.Logf("After relogin - Available: %v, Approved: %v, Serving: %v",
node.GetAvailableRoutes(), node.GetApprovedRoutes(), node.GetSubnetRoutes())
// This is where issue #2896 manifests:
// - Available shows the route (from Hostinfo.RoutableIPs)
// - Approved shows the route (from ApprovedRoutes)
// - BUT Serving (SubnetRoutes/PrimaryRoutes) is EMPTY!
assert.Lenf(c, node.GetAvailableRoutes(), 1,
"Node should have 1 available route after relogin, got %v", node.GetAvailableRoutes())
assert.Lenf(c, node.GetApprovedRoutes(), 1,
"Node should have 1 approved route after relogin, got %v", node.GetApprovedRoutes())
assert.Lenf(c, node.GetSubnetRoutes(), 1,
"BUG #2896: Node should have 1 SERVING route after relogin, got %v", node.GetSubnetRoutes())
assert.Contains(c, node.GetSubnetRoutes(), advertiseRoute,
"BUG #2896: Subnet routes should contain %s after relogin", advertiseRoute)
// Also verify node ID was preserved (same node, not new registration)
assert.Equal(c, initialNodeID, node.GetId(),
"Node ID should be preserved after same-user relogin")
}
}, 30*time.Second, 500*time.Millisecond,
"BUG #2896: routes should remain SERVING after OIDC logout/relogin with same user")
t.Logf("Test completed - verifying issue #2896 fix for OIDC")
}

View File

@@ -2228,9 +2228,9 @@ func TestAutoApproveMultiNetwork(t *testing.T) {
// - Both policy modes (database, file)
// - Both advertiseDuringUp values (true, false)
minimalTestSet := map[string]bool{
"authkey-tag-advertiseduringup-false-pol-database": true, // authkey + database + tag + false
"webauth-user-advertiseduringup-true-pol-file": true, // webauth + file + user + true
"authkey-group-advertiseduringup-false-pol-file": true, // authkey + file + group + false
"authkey-tag-advertiseduringup-false-pol-database": true, // authkey + database + tag + false
"webauth-user-advertiseduringup-true-pol-file": true, // webauth + file + user + true
"authkey-group-advertiseduringup-false-pol-file": true, // authkey + file + group + false
}
for _, tt := range tests {
@@ -2324,7 +2324,11 @@ func TestAutoApproveMultiNetwork(t *testing.T) {
// into a HA node, which isn't something we are testing here.
routerUsernet1, err := scenario.CreateTailscaleNode("head", tsOpts...)
require.NoError(t, err)
defer routerUsernet1.Shutdown()
defer func() {
_, _, err := routerUsernet1.Shutdown()
require.NoError(t, err)
}()
if tt.withURL {
u, err := routerUsernet1.LoginWithURL(headscale.GetEndpoint())
@@ -2333,7 +2337,8 @@ func TestAutoApproveMultiNetwork(t *testing.T) {
body, err := doLoginURL(routerUsernet1.Hostname(), u)
require.NoError(t, err)
scenario.runHeadscaleRegister("user1", body)
err = scenario.runHeadscaleRegister("user1", body)
require.NoError(t, err)
// Wait for the client to sync with the server after webauth registration.
// Unlike authkey login which blocks until complete, webauth registration
@@ -2352,6 +2357,11 @@ func TestAutoApproveMultiNetwork(t *testing.T) {
}
// extra creation end.
// Wait for the node to be fully running before getting its ID
// This is especially important for webauth flow where login is asynchronous
err = routerUsernet1.WaitForRunning(30 * time.Second)
require.NoError(t, err)
routerUsernet1ID := routerUsernet1.MustID()
web := services[0]
@@ -2739,16 +2749,6 @@ func TestAutoApproveMultiNetwork(t *testing.T) {
}
}
func assertTracerouteViaIP(t *testing.T, tr util.Traceroute, ip netip.Addr) {
t.Helper()
require.NotNil(t, tr)
require.True(t, tr.Success)
require.NoError(t, tr.Err)
require.NotEmpty(t, tr.Route)
require.Equal(t, tr.Route[0].IP, ip)
}
// assertTracerouteViaIPWithCollect is a version of assertTracerouteViaIP that works with assert.CollectT.
func assertTracerouteViaIPWithCollect(c *assert.CollectT, tr util.Traceroute, ip netip.Addr) {
assert.NotNil(c, tr)
@@ -2762,30 +2762,6 @@ func assertTracerouteViaIPWithCollect(c *assert.CollectT, tr util.Traceroute, ip
}
}
// requirePeerSubnetRoutes asserts that the peer has the expected subnet routes.
func requirePeerSubnetRoutes(t *testing.T, status *ipnstate.PeerStatus, expected []netip.Prefix) {
t.Helper()
if status.AllowedIPs.Len() <= 2 && len(expected) != 0 {
t.Fatalf("peer %s (%s) has no subnet routes, expected %v", status.HostName, status.ID, expected)
return
}
if len(expected) == 0 {
expected = []netip.Prefix{}
}
got := slicesx.Filter(nil, status.AllowedIPs.AsSlice(), func(p netip.Prefix) bool {
if tsaddr.IsExitRoute(p) {
return true
}
return !slices.ContainsFunc(status.TailscaleIPs, p.Contains)
})
if diff := cmpdiff.Diff(expected, got, util.PrefixComparer, cmpopts.EquateEmpty()); diff != "" {
t.Fatalf("peer %s (%s) subnet routes, unexpected result (-want +got):\n%s", status.HostName, status.ID, diff)
}
}
func SortPeerStatus(a, b *ipnstate.PeerStatus) int {
return cmp.Compare(a.ID, b.ID)
}
@@ -2830,13 +2806,6 @@ func requirePeerSubnetRoutesWithCollect(c *assert.CollectT, status *ipnstate.Pee
}
}
func requireNodeRouteCount(t *testing.T, node *v1.Node, announced, approved, subnet int) {
t.Helper()
require.Lenf(t, node.GetAvailableRoutes(), announced, "expected %q announced routes(%v) to have %d route, had %d", node.GetName(), node.GetAvailableRoutes(), announced, len(node.GetAvailableRoutes()))
require.Lenf(t, node.GetApprovedRoutes(), approved, "expected %q approved routes(%v) to have %d route, had %d", node.GetName(), node.GetApprovedRoutes(), approved, len(node.GetApprovedRoutes()))
require.Lenf(t, node.GetSubnetRoutes(), subnet, "expected %q subnet routes(%v) to have %d route, had %d", node.GetName(), node.GetSubnetRoutes(), subnet, len(node.GetSubnetRoutes()))
}
func requireNodeRouteCountWithCollect(c *assert.CollectT, node *v1.Node, announced, approved, subnet int) {
assert.Lenf(c, node.GetAvailableRoutes(), announced, "expected %q announced routes(%v) to have %d route, had %d", node.GetName(), node.GetAvailableRoutes(), announced, len(node.GetAvailableRoutes()))
assert.Lenf(c, node.GetApprovedRoutes(), approved, "expected %q approved routes(%v) to have %d route, had %d", node.GetName(), node.GetApprovedRoutes(), approved, len(node.GetApprovedRoutes()))