mirror of
https://github.com/juanfont/headscale.git
synced 2025-11-10 22:10:06 -05:00
derp: increase update frequency and harden on failures (#2741)
This commit is contained in:
Kristoffer Dalby
GitHub
@@ -1,16 +1,22 @@
|
||||
package derp
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"hash/crc64"
|
||||
"io"
|
||||
"maps"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/spf13/viper"
|
||||
"gopkg.in/yaml.v3"
|
||||
"tailscale.com/tailcfg"
|
||||
)
|
||||
@@ -79,26 +85,16 @@ func mergeDERPMaps(derpMaps []*tailcfg.DERPMap) *tailcfg.DERPMap {
|
||||
return &result
|
||||
}
|
||||
|
||||
func GetDERPMap(cfg types.DERPConfig) *tailcfg.DERPMap {
|
||||
func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) {
|
||||
var derpMaps []*tailcfg.DERPMap
|
||||
if cfg.DERPMap != nil {
|
||||
derpMaps = append(derpMaps, cfg.DERPMap)
|
||||
}
|
||||
|
||||
for _, path := range cfg.Paths {
|
||||
log.Debug().
|
||||
Str("func", "GetDERPMap").
|
||||
Str("path", path).
|
||||
Msg("Loading DERPMap from path")
|
||||
derpMap, err := loadDERPMapFromPath(path)
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Str("func", "GetDERPMap").
|
||||
Str("path", path).
|
||||
Err(err).
|
||||
Msg("Could not load DERP map from path")
|
||||
|
||||
break
|
||||
return nil, err
|
||||
}
|
||||
|
||||
derpMaps = append(derpMaps, derpMap)
|
||||
@@ -106,26 +102,59 @@ func GetDERPMap(cfg types.DERPConfig) *tailcfg.DERPMap {
|
||||
|
||||
for _, addr := range cfg.URLs {
|
||||
derpMap, err := loadDERPMapFromURL(addr)
|
||||
log.Debug().
|
||||
Str("func", "GetDERPMap").
|
||||
Str("url", addr.String()).
|
||||
Msg("Loading DERPMap from path")
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Str("func", "GetDERPMap").
|
||||
Str("url", addr.String()).
|
||||
Err(err).
|
||||
Msg("Could not load DERP map from path")
|
||||
|
||||
break
|
||||
return nil, err
|
||||
}
|
||||
|
||||
derpMaps = append(derpMaps, derpMap)
|
||||
}
|
||||
|
||||
derpMap := mergeDERPMaps(derpMaps)
|
||||
shuffleDERPMap(derpMap)
|
||||
|
||||
log.Trace().Interface("derpMap", derpMap).Msg("DERPMap loaded")
|
||||
|
||||
return derpMap
|
||||
return derpMap, nil
|
||||
}
|
||||
|
||||
func shuffleDERPMap(dm *tailcfg.DERPMap) {
|
||||
if dm == nil || len(dm.Regions) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for id, region := range dm.Regions {
|
||||
if len(region.Nodes) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
dm.Regions[id] = shuffleRegionNoClone(region)
|
||||
}
|
||||
}
|
||||
|
||||
var crc64Table = crc64.MakeTable(crc64.ISO)
|
||||
|
||||
var (
|
||||
derpRandomOnce sync.Once
|
||||
derpRandomInst *rand.Rand
|
||||
derpRandomMu sync.RWMutex
|
||||
)
|
||||
|
||||
func derpRandom() *rand.Rand {
|
||||
derpRandomOnce.Do(func() {
|
||||
seed := cmp.Or(viper.GetString("dns.base_domain"), time.Now().String())
|
||||
rnd := rand.New(rand.NewSource(0))
|
||||
rnd.Seed(int64(crc64.Checksum([]byte(seed), crc64Table)))
|
||||
derpRandomInst = rnd
|
||||
})
|
||||
return derpRandomInst
|
||||
}
|
||||
|
||||
func resetDerpRandomForTesting() {
|
||||
derpRandomMu.Lock()
|
||||
defer derpRandomMu.Unlock()
|
||||
derpRandomOnce = sync.Once{}
|
||||
derpRandomInst = nil
|
||||
}
|
||||
|
||||
func shuffleRegionNoClone(r *tailcfg.DERPRegion) *tailcfg.DERPRegion {
|
||||
derpRandom().Shuffle(len(r.Nodes), reflect.Swapper(r.Nodes))
|
||||
return r
|
||||
}
|
||||
|
||||
284
hscontrol/derp/derp_test.go
Normal file
284
hscontrol/derp/derp_test.go
Normal file
@@ -0,0 +1,284 @@
|
||||
package derp
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/spf13/viper"
|
||||
"tailscale.com/tailcfg"
|
||||
)
|
||||
|
||||
func TestShuffleDERPMapDeterministic(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
baseDomain string
|
||||
derpMap *tailcfg.DERPMap
|
||||
expected *tailcfg.DERPMap
|
||||
}{
|
||||
{
|
||||
name: "single region with 4 nodes",
|
||||
baseDomain: "test1.example.com",
|
||||
derpMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {
|
||||
RegionID: 1,
|
||||
RegionCode: "nyc",
|
||||
RegionName: "New York City",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "1f", RegionID: 1, HostName: "derp1f.tailscale.com"},
|
||||
{Name: "1g", RegionID: 1, HostName: "derp1g.tailscale.com"},
|
||||
{Name: "1h", RegionID: 1, HostName: "derp1h.tailscale.com"},
|
||||
{Name: "1i", RegionID: 1, HostName: "derp1i.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {
|
||||
RegionID: 1,
|
||||
RegionCode: "nyc",
|
||||
RegionName: "New York City",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "1g", RegionID: 1, HostName: "derp1g.tailscale.com"},
|
||||
{Name: "1f", RegionID: 1, HostName: "derp1f.tailscale.com"},
|
||||
{Name: "1i", RegionID: 1, HostName: "derp1i.tailscale.com"},
|
||||
{Name: "1h", RegionID: 1, HostName: "derp1h.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple regions with nodes",
|
||||
baseDomain: "test2.example.com",
|
||||
derpMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
10: {
|
||||
RegionID: 10,
|
||||
RegionCode: "sea",
|
||||
RegionName: "Seattle",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "10b", RegionID: 10, HostName: "derp10b.tailscale.com"},
|
||||
{Name: "10c", RegionID: 10, HostName: "derp10c.tailscale.com"},
|
||||
{Name: "10d", RegionID: 10, HostName: "derp10d.tailscale.com"},
|
||||
},
|
||||
},
|
||||
2: {
|
||||
RegionID: 2,
|
||||
RegionCode: "sfo",
|
||||
RegionName: "San Francisco",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "2d", RegionID: 2, HostName: "derp2d.tailscale.com"},
|
||||
{Name: "2e", RegionID: 2, HostName: "derp2e.tailscale.com"},
|
||||
{Name: "2f", RegionID: 2, HostName: "derp2f.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
10: {
|
||||
RegionID: 10,
|
||||
RegionCode: "sea",
|
||||
RegionName: "Seattle",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "10b", RegionID: 10, HostName: "derp10b.tailscale.com"},
|
||||
{Name: "10c", RegionID: 10, HostName: "derp10c.tailscale.com"},
|
||||
{Name: "10d", RegionID: 10, HostName: "derp10d.tailscale.com"},
|
||||
},
|
||||
},
|
||||
2: {
|
||||
RegionID: 2,
|
||||
RegionCode: "sfo",
|
||||
RegionName: "San Francisco",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "2f", RegionID: 2, HostName: "derp2f.tailscale.com"},
|
||||
{Name: "2e", RegionID: 2, HostName: "derp2e.tailscale.com"},
|
||||
{Name: "2d", RegionID: 2, HostName: "derp2d.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "large region with many nodes",
|
||||
baseDomain: "test3.example.com",
|
||||
derpMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
4: {
|
||||
RegionID: 4,
|
||||
RegionCode: "fra",
|
||||
RegionName: "Frankfurt",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "4f", RegionID: 4, HostName: "derp4f.tailscale.com"},
|
||||
{Name: "4g", RegionID: 4, HostName: "derp4g.tailscale.com"},
|
||||
{Name: "4h", RegionID: 4, HostName: "derp4h.tailscale.com"},
|
||||
{Name: "4i", RegionID: 4, HostName: "derp4i.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
4: {
|
||||
RegionID: 4,
|
||||
RegionCode: "fra",
|
||||
RegionName: "Frankfurt",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "4f", RegionID: 4, HostName: "derp4f.tailscale.com"},
|
||||
{Name: "4h", RegionID: 4, HostName: "derp4h.tailscale.com"},
|
||||
{Name: "4g", RegionID: 4, HostName: "derp4g.tailscale.com"},
|
||||
{Name: "4i", RegionID: 4, HostName: "derp4i.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "same region different base domain",
|
||||
baseDomain: "different.example.com",
|
||||
derpMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
4: {
|
||||
RegionID: 4,
|
||||
RegionCode: "fra",
|
||||
RegionName: "Frankfurt",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "4f", RegionID: 4, HostName: "derp4f.tailscale.com"},
|
||||
{Name: "4g", RegionID: 4, HostName: "derp4g.tailscale.com"},
|
||||
{Name: "4h", RegionID: 4, HostName: "derp4h.tailscale.com"},
|
||||
{Name: "4i", RegionID: 4, HostName: "derp4i.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
4: {
|
||||
RegionID: 4,
|
||||
RegionCode: "fra",
|
||||
RegionName: "Frankfurt",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "4g", RegionID: 4, HostName: "derp4g.tailscale.com"},
|
||||
{Name: "4i", RegionID: 4, HostName: "derp4i.tailscale.com"},
|
||||
{Name: "4f", RegionID: 4, HostName: "derp4f.tailscale.com"},
|
||||
{Name: "4h", RegionID: 4, HostName: "derp4h.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
viper.Set("dns.base_domain", tt.baseDomain)
|
||||
defer viper.Reset()
|
||||
resetDerpRandomForTesting()
|
||||
|
||||
testMap := tt.derpMap.View().AsStruct()
|
||||
shuffleDERPMap(testMap)
|
||||
|
||||
if diff := cmp.Diff(tt.expected, testMap); diff != "" {
|
||||
t.Errorf("Shuffled DERP map doesn't match expected (-expected +actual):\n%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestShuffleDERPMapEdgeCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
derpMap *tailcfg.DERPMap
|
||||
}{
|
||||
{
|
||||
name: "nil derp map",
|
||||
derpMap: nil,
|
||||
},
|
||||
{
|
||||
name: "empty derp map",
|
||||
derpMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "region with no nodes",
|
||||
derpMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {
|
||||
RegionID: 1,
|
||||
RegionCode: "empty",
|
||||
RegionName: "Empty Region",
|
||||
Nodes: []*tailcfg.DERPNode{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "region with single node",
|
||||
derpMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {
|
||||
RegionID: 1,
|
||||
RegionCode: "single",
|
||||
RegionName: "Single Node Region",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "1a", RegionID: 1, HostName: "derp1a.tailscale.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
shuffleDERPMap(tt.derpMap)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
func TestShuffleDERPMapWithoutBaseDomain(t *testing.T) {
|
||||
viper.Reset()
|
||||
resetDerpRandomForTesting()
|
||||
|
||||
derpMap := &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {
|
||||
RegionID: 1,
|
||||
RegionCode: "test",
|
||||
RegionName: "Test Region",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{Name: "1a", RegionID: 1, HostName: "derp1a.test.com"},
|
||||
{Name: "1b", RegionID: 1, HostName: "derp1b.test.com"},
|
||||
{Name: "1c", RegionID: 1, HostName: "derp1c.test.com"},
|
||||
{Name: "1d", RegionID: 1, HostName: "derp1d.test.com"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
original := derpMap.View().AsStruct()
|
||||
shuffleDERPMap(derpMap)
|
||||
|
||||
if len(derpMap.Regions) != 1 || len(derpMap.Regions[1].Nodes) != 4 {
|
||||
t.Error("Shuffle corrupted DERP map structure")
|
||||
}
|
||||
|
||||
originalNodes := make(map[string]bool)
|
||||
for _, node := range original.Regions[1].Nodes {
|
||||
originalNodes[node.Name] = true
|
||||
}
|
||||
|
||||
shuffledNodes := make(map[string]bool)
|
||||
for _, node := range derpMap.Regions[1].Nodes {
|
||||
shuffledNodes[node.Name] = true
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(originalNodes, shuffledNodes); diff != "" {
|
||||
t.Errorf("Shuffle changed node set (-original +shuffled):\n%s", diff)
|
||||
}
|
||||
}
|
||||
@@ -276,7 +276,7 @@ func DERPProbeHandler(
|
||||
// An example implementation is found here https://derp.tailscale.com/bootstrap-dns
|
||||
// Coordination server is included automatically, since local DERP is using the same DNS Name in d.serverURL.
|
||||
func DERPBootstrapDNSHandler(
|
||||
derpMap *tailcfg.DERPMap,
|
||||
derpMap tailcfg.DERPMapView,
|
||||
) func(http.ResponseWriter, *http.Request) {
|
||||
return func(
|
||||
writer http.ResponseWriter,
|
||||
@@ -287,18 +287,18 @@ func DERPBootstrapDNSHandler(
|
||||
resolvCtx, cancel := context.WithTimeout(req.Context(), time.Minute)
|
||||
defer cancel()
|
||||
var resolver net.Resolver
|
||||
for _, region := range derpMap.Regions {
|
||||
for _, node := range region.Nodes { // we don't care if we override some nodes
|
||||
addrs, err := resolver.LookupIP(resolvCtx, "ip", node.HostName)
|
||||
for _, region := range derpMap.Regions().All() {
|
||||
for _, node := range region.Nodes().All() { // we don't care if we override some nodes
|
||||
addrs, err := resolver.LookupIP(resolvCtx, "ip", node.HostName())
|
||||
if err != nil {
|
||||
log.Trace().
|
||||
Caller().
|
||||
Err(err).
|
||||
Msgf("bootstrap DNS lookup failed %q", node.HostName)
|
||||
Msgf("bootstrap DNS lookup failed %q", node.HostName())
|
||||
|
||||
continue
|
||||
}
|
||||
dnsEntries[node.HostName] = addrs
|
||||
dnsEntries[node.HostName()] = addrs
|
||||
}
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
|
||||
Reference in New Issue
Block a user