Mirrors/headscale
1
0
Fork 0
mirror of https://github.com/juanfont/headscale.git synced 2026-07-21 02:21:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

nix: run each integration test as a NixOS-VM flake check

Browse source 
Each test becomes a hermetic check: a VM boots docker, loads the prebuilt images
and runs that one test (nix/tests/integration.nix), so the suite runs offline as
nix build .#checks.<name>. The generator emits the test list as nix: heavy
multi-subtest tests are split into per-subtest checks, and tests the nested
docker VM can't host are excluded for the legacy pipeline. flake.nix wires the
checks, the images, and the Go build/test/lint checks (with the version JSON on
the embed whitelist).
This commit is contained in:
Kristoffer Dalby 2026-06-25 08:07:22 +00:00
parent bdb9cc510c
commit 70b0c9208b
7 changed files with 637 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

167
.github/workflows/gh-action-integration-generator.go vendored
View file

@ -4,30 +4,49 @@ package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"slices"
"strings"
)
// testsToSplit defines tests that should be split into multiple CI jobs.
// Key is the test function name, value is a list of subtest prefixes.
// Each prefix becomes a separate CI job as "TestName/prefix".
//
// Example: [TestAutoApproveMultiNetwork] has subtests like:
// - TestAutoApproveMultiNetwork/authkey-tag-advertiseduringup-false-pol-database
// - TestAutoApproveMultiNetwork/webauth-user-advertiseduringup-true-pol-file
//
// Splitting by approver type (tag, user, group) creates 6 CI jobs with 4 tests each:
// - TestAutoApproveMultiNetwork/authkey-tag.* (4 tests)
// - TestAutoApproveMultiNetwork/authkey-user.* (4 tests)
// - TestAutoApproveMultiNetwork/authkey-group.* (4 tests)
// - TestAutoApproveMultiNetwork/webauth-tag.* (4 tests)
// - TestAutoApproveMultiNetwork/webauth-user.* (4 tests)
// - TestAutoApproveMultiNetwork/webauth-group.* (4 tests)
//
// This reduces load per CI job (4 tests instead of 12) to avoid infrastructure
// flakiness when running many sequential Docker-based integration tests.
// excludedFromNixChecks lists tests the hermetic nix-VM checks cannot run, with
// the reason. They still run via cmd/hi, where the missing resource exists.
var excludedFromNixChecks = map[string]string{
// Verifies connectivity through Tailscale's *public* DERP relays
// (HEADSCALE_DERP_URLS=controlplane.tailscale.com). A hermetic offline VM
// has no internet and the nix sandbox forbids network.
"TestPingAllByIPPublicDERP": "requires public internet DERP",
// Also routes DERP through Tailscale's public relays: tailscale-rs cannot
// validate the embedded DERP's self-signed cert (WithPublicDERP in the
// test), so it needs the same internet access PublicDERP does.
"TestTailscaleRustAxum": "requires public internet DERP (tailscale-rs)",
// These drive `docker network disconnect`/`connect` to simulate cable pulls.
// In the VM's nested docker, libnetwork intermittently fails the reconnect
// ("failed to set gateway: network is unreachable" — a stale IPAM/endpoint
// reservation), which does not happen against a host docker daemon. They run
// reliably via cmd/hi in the legacy pipeline.
"TestHASubnetRouterFailoverDockerDisconnect": "docker reconnect unreliable in nested VM docker",
"TestHASubnetRouterFailoverBothOfflineCablePull": "docker reconnect unreliable in nested VM docker",
// Ping-driven HA failover: after a router recovers it asserts router 2 stays
// primary and traffic still traceroutes through it. In the VM's nested docker
// that data-plane state never converges (a 4x ScaledTimeout budget did not
// help — behaviour, not timing); it holds against a host docker daemon.
"TestHASubnetRouterPingFailover": "ping-failover data plane doesn't converge in nested VM docker",
}
// testsToSplit defines tests whose subtests are run as separate checks. Key is
// the test function, value is a list of subtest prefixes; each prefix becomes
// its own check as "TestName/prefix.*". TestAutoApproveMultiNetwork fans out
// over approver × policy-mode × advertise (24 subtests); running all of them in
// one nested-docker VM check overruns the timeout, so split by approver into
// 6 checks of 4 — fewer sequential docker scenarios per check.
var testsToSplit = map[string][]string{
"TestAutoApproveMultiNetwork": {
"authkey-tag",
@ -39,26 +58,32 @@ var testsToSplit = map[string][]string{
},
}
// expandTests takes a list of test names and expands any that need splitting
// into multiple subtest patterns.
// expandTests replaces any splittable test with one entry per subtest prefix.
// The ".*" lets the check's "^...$"-wrapped -test.run match the full subtest
// names (e.g. authkey-tag-advertiseduringup-false-pol-database).
func expandTests(tests []string) []string {
var expanded []string
expanded := make([]string, 0, len(tests))
for _, test := range tests {
if prefixes, ok := testsToSplit[test]; ok {
// This test should be split into multiple jobs.
// We append ".*" to each prefix because the CI runner wraps patterns
// with ^...$ anchors. Without ".*", a pattern like "authkey$" wouldn't
// match "authkey-tag-advertiseduringup-false-pol-database".
for _, prefix := range prefixes {
expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix))
}
} else {
prefixes, ok := testsToSplit[test]
if !ok {
expanded = append(expanded, test)
continue
}
for _, prefix := range prefixes {
expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix))
}
}
return expanded
}
// This generator enumerates the integration test functions and writes them as
// nix lists (integration/tests.nix and integration/postgres-tests.nix). The
// flake maps each entry to a NixOS-VM check (sqlite for all, plus a postgres
// variant for the subset) — see nix/tests/integration.nix. One check per test
// function; the GitHub workflow derives its matrix from these checks.
func findTests() []string {
rgBin, err := exec.LookPath("rg")
if err != nil {
@ -89,43 +114,41 @@ func findTests() []string {
return tests
}
func updateYAML(tests []string, jobName string, testPath string) {
testsForYq := fmt.Sprintf("[%s]", strings.Join(tests, ", "))
// writeNixList writes the test function names as a nix list, consumed by
// flake.nix to generate one integration check per test.
func writeNixList(tests []string, nixPath string) {
var b strings.Builder
b.WriteString("# Generated by gh-action-integration-generator.go; do not edit.\n")
b.WriteString("[\n")
for _, test := range tests {
fmt.Fprintf(&b, " %q\n", test)
}
b.WriteString("]\n")
yqCommand := fmt.Sprintf(
"yq eval '.jobs.%s.strategy.matrix.test = %s' %s -i",
jobName,
testsForYq,
testPath,
)
cmd := exec.Command("bash", "-c", yqCommand)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
log.Printf("stdout: %s", stdout.String())
log.Printf("stderr: %s", stderr.String())
log.Fatalf("failed to run yq command: %s", err)
if err := os.WriteFile(nixPath, []byte(b.String()), 0o644); err != nil {
log.Fatalf("failed to write %s: %s", nixPath, err)
}
fmt.Printf("YAML file (%s) job %s updated successfully\n", testPath, jobName)
fmt.Printf("Nix list (%s) updated successfully\n", nixPath)
}
func main() {
tests := findTests()
// Expand tests that should be split into multiple jobs
expandedTests := expandTests(tests)
tests = slices.DeleteFunc(tests, func(name string) bool {
if reason, ok := excludedFromNixChecks[name]; ok {
log.Printf("excluding %s from nix checks: %s", name, reason)
return true
}
return false
})
quotedTests := make([]string, len(expandedTests))
for i, test := range expandedTests {
quotedTests[i] = fmt.Sprintf("\"%s\"", test)
}
// Split heavy tests into per-subtest checks (after exclusion, so an excluded
// test is never expanded).
tests = expandTests(tests)
// Define selected tests for PostgreSQL
// Selected tests also run against PostgreSQL to verify database
// compatibility (a postgres-variant check per name).
postgresTestNames := []string{
"TestACLAllowUserDst",
"TestPingAllByIP",
@ -134,12 +157,30 @@ func main() {
"TestSubnetRouterMultiNetwork",
}
quotedPostgresTests := make([]string, len(postgresTestNames))
for i, test := range postgresTestNames {
quotedPostgresTests[i] = fmt.Sprintf("\"%s\"", test)
// Tests excluded from the hermetic nix checks still need to run somewhere,
// so emit them for the legacy cmd/hi GitHub-Actions matrix (real internet).
excluded := make([]string, 0, len(excludedFromNixChecks))
for name := range excludedFromNixChecks {
excluded = append(excluded, name)
}
slices.Sort(excluded)
writeNixList(tests, "../../integration/tests.nix")
writeNixList(postgresTestNames, "../../integration/postgres-tests.nix")
writeJSONList(excluded, "../../integration/excluded-tests.json")
}
// writeJSONList writes the names as a JSON array, consumed as a GitHub-Actions
// matrix by the legacy cmd/hi workflow.
func writeJSONList(names []string, path string) {
data, err := json.MarshalIndent(names, "", " ")
if err != nil {
log.Fatalf("marshalling %s: %s", path, err)
}
// Update both SQLite and PostgreSQL job matrices
updateYAML(quotedTests, "sqlite", "./test-integration.yaml")
updateYAML(quotedPostgresTests, "postgres", "./test-integration.yaml")
if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil {
log.Fatalf("failed to write %s: %s", path, err)
}
fmt.Printf("JSON list (%s) updated successfully\n", path)
}

70
flake.lock generated
View file

@ -22,6 +22,22 @@
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1767039857,
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
@ -74,11 +90,28 @@
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1772736753,
"narHash": "sha256-au/m3+EuBLoSzWUCb64a/MZq6QUtOV8oC0D9tY2scPQ=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "917fec990948658ef1ccd07cef2a1ef060786846",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-checks": "flake-checks",
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs"
"nixpkgs": "nixpkgs",
"tailscale-head": "tailscale-head"
}
},
"systems": {
@ -111,6 +144,41 @@
"type": "github"
}
},
"systems_3": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"tailscale-head": {
"inputs": {
"flake-compat": "flake-compat",
"nixpkgs": "nixpkgs_2",
"systems": "systems_3"
},
"locked": {
"lastModified": 1782250635,
"narHash": "sha256-y4/EAApBPekjWz2cP/9RexTBkiSAXsdOHWnhm/no/4I=",
"owner": "tailscale",
"repo": "tailscale",
"rev": "d4f2917c1bfd27529ec7997d08a8f1aa9ea903f2",
"type": "github"
},
"original": {
"owner": "tailscale",
"repo": "tailscale",
"type": "github"
}
},
"treefmt-nix": {
"inputs": {
"nixpkgs": [

138
flake.nix
View file

@ -9,10 +9,18 @@
# once it ships go_1_26 >= 1.26.4.
nixpkgs.url = "github:NixOS/nixpkgs/staging-next-26.05";
flake-utils.url = "github:numtide/flake-utils";
# Reusable Go flake checks (build/test/lint/format); CI runs them via
# `nix build .#checks.<system>.<name>` instead of bespoke per-tool steps.
flake-checks.url = "github:kradalby/flake-checks";
flake-checks.inputs.nixpkgs.follows = "nixpkgs";
# Tailscale HEAD, built from source for the integration test client image.
# Bump with `nix flake update tailscale-head` to track the latest main.
# It is a flake; we still build our own derivation (we need derper +
# containerboot, which its default package omits) but read its committed
# vendorHash from flakehashes.json so the bump carries the hash.
tailscale-head.url = "github:tailscale/tailscale";
};
outputs =
@ -20,6 +28,7 @@
, nixpkgs
, flake-utils
, flake-checks
, tailscale-head
, ...
}:
let
@ -39,8 +48,27 @@
# Go 1.26 builder; resolves to Go 1.26.4 from the pinned nixpkgs.
buildGo = pkgs.buildGo126Module;
vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri;
# Go source with the non-Go scaffolding filtered out, so editing the
# nix integration harness, CI workflows, docs or packaging does not
# rebuild the (slow) integration test binary or container images.
# Only changes to actual Go source invalidate them.
goSrc = pkgs.lib.fileset.toSource {
root = ./.;
fileset = pkgs.lib.fileset.difference ./. (pkgs.lib.fileset.unions [
./nix
./.github
./docs
./packaging
./flake.nix
./flake.lock
./flakehashes.json
]);
};
in
{
headscale-go-src = goSrc;
headscale = buildGo {
pname = "headscale";
version = headscaleVersion;
@ -71,6 +99,34 @@
subPackages = [ "cmd/hi" ];
};
# The integration suite compiled into a single test binary, run later
# inside a per-test NixOS VM check (see nix/tests/integration.nix).
# Compiled once and shared by every integration check.
integration-test-bin = buildGo {
pname = "headscale-integration-test";
# Fixed (not the commit rev) so the binary is content-addressed:
# identical Go source → identical store path → shared-cache hit
# across commits. The version string is irrelevant for a test binary.
version = "integration";
src = goSrc;
inherit vendorHash;
doCheck = false;
buildPhase = ''
runHook preBuild
go test -c -o integration.test ./integration
runHook postBuild
'';
installPhase = ''
runHook preInstall
install -Dm755 integration.test $out/bin/integration.test
runHook postInstall
'';
};
# Build golangci-lint with stock Go 1.26 (upstream uses hardcoded Go
# version); it does not build against the pinned 1.26.4.
golangci-lint = buildGo rec {
@ -200,7 +256,12 @@
vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri;
goPkg = pkgs.go_1_26;
# //go:embed targets and test-read files outside the default whitelist.
embedDirs = [ ./hscontrol/assets ./hscontrol/db/schema.sql ./config-example.yaml ];
embedDirs = [
./hscontrol/assets
./hscontrol/db/schema.sql
./config-example.yaml
./integration/tailscale-versions.json
];
extraSrc = [
./hscontrol/testdata
./hscontrol/types/testdata
@ -239,6 +300,45 @@
fmtExclude = [ ./gen ./docs ];
});
};
# Nix-built container images for the integration tests, plus the
# per-test VM check factory. Linux-only (dockerTools + nixosTest).
integrationImages = import ./nix/images.nix {
inherit pkgs;
buildGoModule = pkgs.buildGo126Module;
tailscaleSrc = tailscale-head;
};
mkIntegrationCheck = { name, testFilter ? name, postgres ? false }:
pkgs.testers.nixosTest (import ./nix/tests/integration.nix {
inherit name testFilter pkgs postgres;
inherit (integrationImages)
headscaleImage tailscaleImage postgresImage tailscaleVersionImages;
testBin = pkgs.integration-test-bin;
# goSrc (not the full tree) so a check's result stays cached when
# only nix/CI/docs change — only real Go/integration changes (which
# also rebuild testBin) invalidate it. Maximises shared-cache reuse.
src = pkgs.headscale-go-src;
});
# One sqlite check per test (full matrix), plus a postgres variant for
# the postgres subset — exactly the sqlite+postgres jobs the generator
# produces for the GitHub workflow.
integrationChecks =
let
# Split-test entries (TestAutoApproveMultiNetwork/authkey-tag.*) carry
# / and * in their -test.run filter; sanitise those into a valid check
# name while keeping the raw filter for -test.run.
sanitize = builtins.replaceStrings [ "/" "." "*" ] [ "-" "" "" ];
mkCheck = postgres: filter:
pkgs.lib.nameValuePair
"integration-${sanitize filter}${pkgs.lib.optionalString postgres "-pg"}"
(mkIntegrationCheck { name = sanitize filter; testFilter = filter; inherit postgres; });
in
pkgs.lib.listToAttrs (
map (mkCheck false) (import ./integration/tests.nix)
++ map (mkCheck true) (import ./integration/postgres-tests.nix)
);
in
{
# `nix develop`
@ -259,6 +359,14 @@
cat go.mod | ${pkgs.ripgrep}/bin/rg "\t" | ${pkgs.ripgrep}/bin/rg -v indirect | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.findutils}/bin/xargs go get -u
go mod tidy
'')
# Regenerate integration/tailscale-versions.json after a capver bump
# so the offline checks pin the tailscale images the suite requests.
# It is a //go:generate step in the integration package (runs with
# capver under `make generate`); this just scopes it to the pins.
(pkgs.writeShellScriptBin "update-integration-images" ''
exec go generate ./integration/...
'')
];
shellHook = ''
@ -272,6 +380,28 @@
inherit headscale;
inherit headscale-docker;
default = headscale;
}
// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux {
headscale-integration-image = integrationImages.headscaleImage;
tailscale-integration-image = integrationImages.tailscaleImage;
postgres-integration-image = integrationImages.postgresImage;
inherit (pkgs) integration-test-bin;
# All shared inputs every integration check pulls in: the test binary
# and every image. CI builds this once (a "prime" job) so it lands in
# the shared nix cache before the per-test matrix fans out — the
# matrix jobs then substitute it instead of each rebuilding.
integration-deps = pkgs.linkFarm "integration-deps" (
[
{ name = "test-bin"; path = pkgs.integration-test-bin; }
{ name = "headscale"; path = integrationImages.headscaleImage; }
{ name = "tailscale-head"; path = integrationImages.tailscaleImage; }
{ name = "postgres"; path = integrationImages.postgresImage; }
]
++ pkgs.lib.imap0
(i: img: { name = "ts-version-${toString i}"; path = img; })
integrationImages.tailscaleVersionImages
);
};
# `nix run`
@ -287,6 +417,10 @@
}
# The Go build/test checks are gated to Linux: parts of the tree are
# Linux-specific and the pure unit subset is validated by CI.
// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks;
// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks
# Integration checks only on x86_64-linux: nixosTest needs KVM and the
# pinned version images are amd64 (nix/tailscale-versions.nix). aarch64
# would need arch-specific pins.
// pkgs.lib.optionalAttrs (system == "x86_64-linux") integrationChecks;
});
}

7
integration/excluded-tests.json Normal file
View file

@ -0,0 +1,7 @@
[
"TestHASubnetRouterFailoverBothOfflineCablePull",
"TestHASubnetRouterFailoverDockerDisconnect",
"TestHASubnetRouterPingFailover",
"TestPingAllByIPPublicDERP",
"TestTailscaleRustAxum"
]

8
integration/postgres-tests.nix Normal file
View file

@ -0,0 +1,8 @@
# Generated by gh-action-integration-generator.go; do not edit.
[
"TestACLAllowUserDst"
"TestPingAllByIP"
"TestEphemeral2006DeletedTooQuickly"
"TestPingAllByIPManyUpDown"
"TestSubnetRouterMultiNetwork"
]

156
integration/tests.nix Normal file
View file

@ -0,0 +1,156 @@
# Generated by gh-action-integration-generator.go; do not edit.
[
"TestACLHostsInNetMapTable"
"TestACLAllowUser80Dst"
"TestACLDenyAllPort80"
"TestACLAllowUserDst"
"TestACLAllowStarDst"
"TestACLNamedHostsCanReachBySubnet"
"TestACLNamedHostsCanReach"
"TestACLDevice1CanAccessDevice2"
"TestPolicyUpdateWhileRunningWithCLIInDatabase"
"TestACLAutogroupMember"
"TestACLAutogroupTagged"
"TestACLAutogroupSelf"
"TestACLPolicyPropagationOverTime"
"TestACLTagPropagation"
"TestACLTagPropagationPortSpecific"
"TestACLGroupWithUnknownUser"
"TestACLGroupAfterUserDeletion"
"TestACLGroupDeletionExactReproduction"
"TestACLDynamicUnknownUserAddition"
"TestACLDynamicUnknownUserRemoval"
"TestAPIAuthenticationBypass"
"TestAPIAuthenticationBypassCurl"
"TestRemoteCLIAuthenticationBypass"
"TestCLIWithConfigAuthenticationBypass"
"TestAuthKeyLogoutAndReloginSameUser"
"TestAuthKeyLogoutAndReloginNewUser"
"TestAuthKeyLogoutAndReloginSameUserExpiredKey"
"TestAuthKeyDeleteKey"
"TestAuthKeyLogoutAndReloginRoutesPreserved"
"TestOIDCAuthenticationPingAll"
"TestOIDCExpireNodesBasedOnTokenExpiry"
"TestOIDC024UserCreation"
"TestOIDCAuthenticationWithPKCE"
"TestOIDCReloginSameNodeNewUser"
"TestOIDCFollowUpUrl"
"TestOIDCMultipleOpenedLoginUrls"
"TestOIDCReloginSameNodeSameUser"
"TestOIDCExpiryAfterRestart"
"TestOIDCACLPolicyOnJoin"
"TestOIDCReloginSameUserRoutesPreserved"
"TestAuthWebFlowAuthenticationPingAll"
"TestAuthWebFlowLogoutAndReloginSameUser"
"TestAuthWebFlowLogoutAndReloginNewUser"
"TestApiKeyCommand"
"TestApiKeyCommandValidation"
"TestAuthCommandValidation"
"TestNodeCommand"
"TestNodeExpireCommand"
"TestNodeRenameCommand"
"TestPreAuthKeyCorrectUserLoggedInCommand"
"TestTaggedNodesCLIOutput"
"TestNodeExpireFlagsCommand"
"TestNodeCommandValidation"
"TestNodeTagCommand"
"TestNodeRouteCommands"
"TestNodeBackfillIPsCommand"
"TestPolicyCheckCommand"
"TestSSHTestsRejectFailingPolicy"
"TestPolicyCommand"
"TestPolicyBrokenConfigCommand"
"TestPreAuthKeyCommand"
"TestPreAuthKeyCommandWithoutExpiry"
"TestPreAuthKeyCommandReusableEphemeral"
"TestPreAuthKeyDeleteCommand"
"TestPreAuthKeyCommandValidation"
"TestServerInfoCommands"
"TestUserCommand"
"TestUserCreateCommand"
"TestUserCommandValidation"
"TestDERPVerifyEndpoint"
"TestResolveMagicDNS"
"TestResolveMagicDNSExtraRecordsPath"
"TestDERPServerScenario"
"TestDERPServerWebsocketScenario"
"TestPingAllByIP"
"TestEphemeral"
"TestEphemeralInAlternateTimezone"
"TestEphemeral2006DeletedTooQuickly"
"TestPingAllByHostname"
"TestTaildrop"
"TestUpdateHostnameFromClient"
"TestExpireNode"
"TestSetNodeExpiryInFuture"
"TestDisableNodeExpiry"
"TestNodeOnlineStatus"
"TestPingAllByIPManyUpDown"
"Test2118DeletingOnlineNodePanics"
"TestGrantCapRelay"
"TestGrantCapDrive"
"TestEnablingRoutes"
"TestHASubnetRouterFailover"
"TestSubnetRouteACL"
"TestEnablingExitRoutes"
"TestExitRoutesWithAutogroupInternetACL"
"TestSubnetRouterMultiNetwork"
"TestSubnetRouterMultiNetworkExitNode"
"TestAutoApproveMultiNetwork/authkey-tag.*"
"TestAutoApproveMultiNetwork/authkey-user.*"
"TestAutoApproveMultiNetwork/authkey-group.*"
"TestAutoApproveMultiNetwork/webauth-tag.*"
"TestAutoApproveMultiNetwork/webauth-user.*"
"TestAutoApproveMultiNetwork/webauth-group.*"
"TestSubnetRouteACLFiltering"
"TestGrantViaSubnetSteering"
"TestHASubnetRouterFailoverBothOffline"
"TestHeadscale"
"TestTailscaleNodesJoiningHeadcale"
"TestSSHOneUserToAll"
"TestSSHMultipleUsersAllToAll"
"TestSSHNoSSHConfigured"
"TestSSHIsBlockedInACL"
"TestSSHUserOnlyIsolation"
"TestSSHAutogroupSelf"
"TestSSHOneUserToOneCheckModeCLI"
"TestSSHOneUserToOneCheckModeOIDC"
"TestSSHCheckModeUnapprovedTimeout"
"TestSSHCheckModeCheckPeriodCLI"
"TestSSHCheckModeAutoApprove"
"TestSSHCheckModeSessionLossReDelegates"
"TestSSHCheckModeNegativeCLI"
"TestSSHLocalpart"
"TestTagsAuthKeyWithTagRequestDifferentTag"
"TestTagsAuthKeyWithTagNoAdvertiseFlag"
"TestTagsAuthKeyWithTagCannotAddViaCLI"
"TestTagsAuthKeyWithTagCannotChangeViaCLI"
"TestTagsAuthKeyWithTagAdminOverrideReauthPreserves"
"TestTagsAuthKeyWithTagCLICannotModifyAdminTags"
"TestTagsAuthKeyWithoutTagCannotRequestTags"
"TestTagsAuthKeyWithoutTagRegisterNoTags"
"TestTagsAuthKeyWithoutTagCannotAddViaCLI"
"TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset"
"TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise"
"TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag"
"TestTagsUserLoginOwnedTagAtRegistration"
"TestTagsUserLoginNonExistentTagAtRegistration"
"TestTagsUserLoginUnownedTagAtRegistration"
"TestTagsUserLoginAddTagViaCLIReauth"
"TestTagsUserLoginRemoveTagViaCLIReauth"
"TestTagsUserLoginCLINoOpAfterAdminAssignment"
"TestTagsUserLoginCLICannotRemoveAdminTags"
"TestTagsAuthKeyWithTagRequestNonExistentTag"
"TestTagsAuthKeyWithTagRequestUnownedTag"
"TestTagsAuthKeyWithoutTagRequestNonExistentTag"
"TestTagsAuthKeyWithoutTagRequestUnownedTag"
"TestTagsAdminAPICannotSetNonExistentTag"
"TestTagsAdminAPICanSetUnownedTag"
"TestTagsAdminAPICannotRemoveAllTags"
"TestTagsIssue2978ReproTagReplacement"
"TestTagsAdminAPICannotSetInvalidFormat"
"TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags"
"TestTagsAuthKeyWithoutUserInheritsTags"
"TestTagsAuthKeyWithoutUserRejectsAdvertisedTags"
"TestTagsAuthKeyConvertToUserViaCLIRegister"
]

157
nix/tests/integration.nix Normal file
View file

@ -0,0 +1,157 @@
# Per-test integration check: a NixOS VM that boots docker, loads the nix-built
# + pinned images (full tailscale version matrix), and runs the prebuilt
# integration test binary filtered to a single test. Hermetic equivalent of one
# `go run ./cmd/hi run <TestName>` job. `postgres = true` produces the postgres
# variant (HEADSCALE_INTEGRATION_POSTGRES=1) for the postgres subset.
#
# The suite runs *inside* a container named headscale-test-suite-<runID> with
# the docker socket mounted (docker-out-of-docker): scenario.go attaches that
# container to each test network, and IntegrationSkip gates on /.dockerenv. We
# reuse the headscale image as the runner base and bind-mount the host nix store
# so the test binary's closure resolves.
{ name
, testFilter ? name
, pkgs
, headscaleImage
, tailscaleImage
, postgresImage
, tailscaleVersionImages
, testBin
, src
, postgres ? false
}:
let
lib = pkgs.lib;
# The test function name, dropping any "/subtest" filter on a split check, so
# the skip-guard below matches a whole-function skip without tripping on a
# legitimately-skipped subtest.
testFunc = builtins.head (lib.splitString "/" testFilter);
# >=6 chars and DNS-safe: tsic/hsic derive container hostnames from the last
# 6 chars of the run ID (runID[len-6:]), which panics on shorter ids.
runID = "nixcheck";
runnerName = "headscale-test-suite-${runID}";
# Images this check must have locally before the suite runs. Every test may
# fan out over the full tailscale version matrix (MustTestVersions); the head
# image doubles as the DERP server; the postgres variant adds postgres.
images =
[ headscaleImage tailscaleImage ]
++ tailscaleVersionImages
++ lib.optional postgres postgresImage;
# Loading ~8 images dominates wall time; do it in parallel (they share layers,
# so docker dedups) and fail if any load fails.
loadImages = pkgs.writeShellScript "load-integration-images" ''
set -u
pids=""
${lib.concatMapStringsSep "\n" (img: ''docker load -i ${img} & pids="$pids $!"'') images}
rc=0
for p in $pids; do wait "$p" || rc=1; done
exit "$rc"
'';
pgEnv = lib.optionalString postgres
"-e HEADSCALE_INTEGRATION_POSTGRES=1 -e HEADSCALE_INTEGRATION_POSTGRES_IMAGE=postgres:16 ";
in
{
name = "integration-${name}${lib.optionalString postgres "-pg"}";
nodes.machine = { ... }: {
virtualisation.docker.enable = true;
# Match the integration CI: Docker 29's default containerd snapshotter +
# overlayfs regresses image load/exec for these images; the classic
# overlay2 graph driver is the known-good path.
virtualisation.docker.daemon.settings = {
storage-driver = "overlay2";
features.containerd-snapshotter = false;
};
# tailscaled needs /dev/net/tun for its --tun=tsdev device.
boot.kernelModules = [ "tun" ];
# The VM's dhcpcd otherwise manages docker's bridges/veths: as containers
# and the cable-pull tests churn interfaces it deletes their routes and
# races libnetwork's gateway setup, surfacing as
# "API error (500): updating gateway endpoint: failed to set gateway:
# network is unreachable" on reconnect (TestHASubnetRouterFailover*Disconnect
# /CablePull) and intermittent container-startup flakes. Keep it off them.
networking.dhcpcd.denyInterfaces = [ "veth*" "br-*" "docker*" ];
# Trim the guest toward a tiny profile: this VM only needs to boot, run
# docker, and exec one test binary. Measured boot is 18.7s (1.7s kernel +
# 3.9s initrd + 13.1s userspace); the userspace critical chain is
# dhcpcd (6.1s) -> network-online.target -> docker.service (+3.5s), i.e. the
# remaining cost is functional (docker waits for the network), not fat to
# trim. documentation + timesyncd off shave the cheap, avoidable part.
#
# microvm.nix was evaluated against this and rejected: it still needs a full
# docker daemon + nested KVM, and boot is only ~10-15% of the 2-3 min
# image-load + test runtime, so a new flake input can't pay itself back. The
# one real lever left, dropping docker's network-online wait (~6s), sits
# right next to the dhcpcd interface races handled below and isn't worth
# destabilising the matrix for.
documentation.enable = false;
services.timesyncd.enable = false;
# Sized for the full version matrix (a test can spawn 12+ tailscale
# containers + headscale + the runner) while still fitting a standard
# 4-core/16 GB GitHub runner. Disk-backed docker (not tmpfs) keeps RAM
# headroom on the runner; the slim source copy below is the bigger win.
virtualisation.memorySize = 8192;
virtualisation.cores = 4;
virtualisation.diskSize = 12288;
};
testScript = ''
start_all()
machine.wait_for_unit("docker.service")
# Load all matrix images in parallel (store paths visible inside the VM).
machine.succeed("${loadImages}", timeout=600)
# Writable copy of just the integration package (the suite writes
# control_logs/ under CWD and reads testdata relative to it). Only this
# subtree is needed at runtime — the repo-root build context is only used
# for image builds, which we bypass with prebuilt images.
machine.succeed("mkdir -p /tmp/hs && cp -r ${src}/integration /tmp/hs/integration && chmod -R u+w /tmp/hs")
# hsic/tsic save artifacts (and tsic.Status writes <host>_status.json) under
# /tmp/control; cmd/hi mounts this dir. Without it every Status() errors on
# the write and the suite never observes NeedsLogin.
machine.succeed("mkdir -p /tmp/control")
# Run the suite inside a container (docker-out-of-docker), mirroring cmd/hi:
# - name matches scenario.go's testSuiteName so it can join test networks
# - docker socket mounted so sibling containers spawn on this daemon
# - host nix store mounted so the test binary's closure resolves
out = machine.succeed(
"docker run --rm --name ${runnerName} "
"-v /var/run/docker.sock:/var/run/docker.sock "
"-v /nix/store:/nix/store:ro "
"-v /tmp/hs:/tmp/hs -w /tmp/hs/integration "
"-v /tmp/control:/tmp/control "
"-e HEADSCALE_INTEGRATION_HEADSCALE_IMAGE=headscale:nix "
"-e HEADSCALE_INTEGRATION_TAILSCALE_IMAGE=tailscale-head:nix "
"-e HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE=tailscale-head:nix "
"-e HEADSCALE_INTEGRATION_DERPER_IMAGE=tailscale-head:nix "
"${pgEnv}"
"-e HEADSCALE_INTEGRATION_RUN_ID=${runID} "
"-e CI=1 "
# The nested-docker VM converges slower than a host docker daemon; widen
# the ScaledTimeout budgets (subnet-route convergence, HA cycles) past 2x.
"-e HEADSCALE_INTEGRATION_TIMEOUT_SCALE=4 "
"-e SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt "
"headscale:nix "
"${testBin}/bin/integration.test -test.run '^${testFilter}$' -test.v -test.timeout=20m",
timeout=1800,
)
print(out)
# A skipped or unmatched test still exits 0; guard against a false green.
assert "no tests to run" not in out, "filter ${testFilter} matched nothing"
assert "--- SKIP: ${testFunc} " not in out, "test ${testFunc} was skipped (not in container?)"
'';
}