diff --git a/.github/workflows/gh-action-integration-generator.go b/.github/workflows/gh-action-integration-generator.go index 5bb8318bd..cc9c069c0 100644 --- a/.github/workflows/gh-action-integration-generator.go +++ b/.github/workflows/gh-action-integration-generator.go @@ -4,30 +4,49 @@ package main import ( "bytes" + "encoding/json" "fmt" "log" + "os" "os/exec" + "slices" "strings" ) -// testsToSplit defines tests that should be split into multiple CI jobs. -// Key is the test function name, value is a list of subtest prefixes. -// Each prefix becomes a separate CI job as "TestName/prefix". -// -// Example: [TestAutoApproveMultiNetwork] has subtests like: -// - TestAutoApproveMultiNetwork/authkey-tag-advertiseduringup-false-pol-database -// - TestAutoApproveMultiNetwork/webauth-user-advertiseduringup-true-pol-file -// -// Splitting by approver type (tag, user, group) creates 6 CI jobs with 4 tests each: -// - TestAutoApproveMultiNetwork/authkey-tag.* (4 tests) -// - TestAutoApproveMultiNetwork/authkey-user.* (4 tests) -// - TestAutoApproveMultiNetwork/authkey-group.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-tag.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-user.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-group.* (4 tests) -// -// This reduces load per CI job (4 tests instead of 12) to avoid infrastructure -// flakiness when running many sequential Docker-based integration tests. +// excludedFromNixChecks lists tests the hermetic nix-VM checks cannot run, with +// the reason. They still run via cmd/hi, where the missing resource exists. +var excludedFromNixChecks = map[string]string{ + // Verifies connectivity through Tailscale's *public* DERP relays + // (HEADSCALE_DERP_URLS=controlplane.tailscale.com). A hermetic offline VM + // has no internet and the nix sandbox forbids network. + "TestPingAllByIPPublicDERP": "requires public internet DERP", + + // Also routes DERP through Tailscale's public relays: tailscale-rs cannot + // validate the embedded DERP's self-signed cert (WithPublicDERP in the + // test), so it needs the same internet access PublicDERP does. + "TestTailscaleRustAxum": "requires public internet DERP (tailscale-rs)", + + // These drive `docker network disconnect`/`connect` to simulate cable pulls. + // In the VM's nested docker, libnetwork intermittently fails the reconnect + // ("failed to set gateway: network is unreachable" — a stale IPAM/endpoint + // reservation), which does not happen against a host docker daemon. They run + // reliably via cmd/hi in the legacy pipeline. + "TestHASubnetRouterFailoverDockerDisconnect": "docker reconnect unreliable in nested VM docker", + "TestHASubnetRouterFailoverBothOfflineCablePull": "docker reconnect unreliable in nested VM docker", + + // Ping-driven HA failover: after a router recovers it asserts router 2 stays + // primary and traffic still traceroutes through it. In the VM's nested docker + // that data-plane state never converges (a 4x ScaledTimeout budget did not + // help — behaviour, not timing); it holds against a host docker daemon. + "TestHASubnetRouterPingFailover": "ping-failover data plane doesn't converge in nested VM docker", +} + +// testsToSplit defines tests whose subtests are run as separate checks. Key is +// the test function, value is a list of subtest prefixes; each prefix becomes +// its own check as "TestName/prefix.*". TestAutoApproveMultiNetwork fans out +// over approver × policy-mode × advertise (24 subtests); running all of them in +// one nested-docker VM check overruns the timeout, so split by approver into +// 6 checks of 4 — fewer sequential docker scenarios per check. var testsToSplit = map[string][]string{ "TestAutoApproveMultiNetwork": { "authkey-tag", @@ -39,26 +58,32 @@ var testsToSplit = map[string][]string{ }, } -// expandTests takes a list of test names and expands any that need splitting -// into multiple subtest patterns. +// expandTests replaces any splittable test with one entry per subtest prefix. +// The ".*" lets the check's "^...$"-wrapped -test.run match the full subtest +// names (e.g. authkey-tag-advertiseduringup-false-pol-database). func expandTests(tests []string) []string { - var expanded []string + expanded := make([]string, 0, len(tests)) for _, test := range tests { - if prefixes, ok := testsToSplit[test]; ok { - // This test should be split into multiple jobs. - // We append ".*" to each prefix because the CI runner wraps patterns - // with ^...$ anchors. Without ".*", a pattern like "authkey$" wouldn't - // match "authkey-tag-advertiseduringup-false-pol-database". - for _, prefix := range prefixes { - expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix)) - } - } else { + prefixes, ok := testsToSplit[test] + if !ok { expanded = append(expanded, test) + continue + } + + for _, prefix := range prefixes { + expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix)) } } + return expanded } +// This generator enumerates the integration test functions and writes them as +// nix lists (integration/tests.nix and integration/postgres-tests.nix). The +// flake maps each entry to a NixOS-VM check (sqlite for all, plus a postgres +// variant for the subset) — see nix/tests/integration.nix. One check per test +// function; the GitHub workflow derives its matrix from these checks. + func findTests() []string { rgBin, err := exec.LookPath("rg") if err != nil { @@ -89,43 +114,41 @@ func findTests() []string { return tests } -func updateYAML(tests []string, jobName string, testPath string) { - testsForYq := fmt.Sprintf("[%s]", strings.Join(tests, ", ")) +// writeNixList writes the test function names as a nix list, consumed by +// flake.nix to generate one integration check per test. +func writeNixList(tests []string, nixPath string) { + var b strings.Builder + b.WriteString("# Generated by gh-action-integration-generator.go; do not edit.\n") + b.WriteString("[\n") + for _, test := range tests { + fmt.Fprintf(&b, " %q\n", test) + } + b.WriteString("]\n") - yqCommand := fmt.Sprintf( - "yq eval '.jobs.%s.strategy.matrix.test = %s' %s -i", - jobName, - testsForYq, - testPath, - ) - cmd := exec.Command("bash", "-c", yqCommand) - - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err := cmd.Run() - if err != nil { - log.Printf("stdout: %s", stdout.String()) - log.Printf("stderr: %s", stderr.String()) - log.Fatalf("failed to run yq command: %s", err) + if err := os.WriteFile(nixPath, []byte(b.String()), 0o644); err != nil { + log.Fatalf("failed to write %s: %s", nixPath, err) } - fmt.Printf("YAML file (%s) job %s updated successfully\n", testPath, jobName) + fmt.Printf("Nix list (%s) updated successfully\n", nixPath) } func main() { tests := findTests() - // Expand tests that should be split into multiple jobs - expandedTests := expandTests(tests) + tests = slices.DeleteFunc(tests, func(name string) bool { + if reason, ok := excludedFromNixChecks[name]; ok { + log.Printf("excluding %s from nix checks: %s", name, reason) + return true + } + return false + }) - quotedTests := make([]string, len(expandedTests)) - for i, test := range expandedTests { - quotedTests[i] = fmt.Sprintf("\"%s\"", test) - } + // Split heavy tests into per-subtest checks (after exclusion, so an excluded + // test is never expanded). + tests = expandTests(tests) - // Define selected tests for PostgreSQL + // Selected tests also run against PostgreSQL to verify database + // compatibility (a postgres-variant check per name). postgresTestNames := []string{ "TestACLAllowUserDst", "TestPingAllByIP", @@ -134,12 +157,30 @@ func main() { "TestSubnetRouterMultiNetwork", } - quotedPostgresTests := make([]string, len(postgresTestNames)) - for i, test := range postgresTestNames { - quotedPostgresTests[i] = fmt.Sprintf("\"%s\"", test) + // Tests excluded from the hermetic nix checks still need to run somewhere, + // so emit them for the legacy cmd/hi GitHub-Actions matrix (real internet). + excluded := make([]string, 0, len(excludedFromNixChecks)) + for name := range excludedFromNixChecks { + excluded = append(excluded, name) + } + slices.Sort(excluded) + + writeNixList(tests, "../../integration/tests.nix") + writeNixList(postgresTestNames, "../../integration/postgres-tests.nix") + writeJSONList(excluded, "../../integration/excluded-tests.json") +} + +// writeJSONList writes the names as a JSON array, consumed as a GitHub-Actions +// matrix by the legacy cmd/hi workflow. +func writeJSONList(names []string, path string) { + data, err := json.MarshalIndent(names, "", " ") + if err != nil { + log.Fatalf("marshalling %s: %s", path, err) } - // Update both SQLite and PostgreSQL job matrices - updateYAML(quotedTests, "sqlite", "./test-integration.yaml") - updateYAML(quotedPostgresTests, "postgres", "./test-integration.yaml") + if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil { + log.Fatalf("failed to write %s: %s", path, err) + } + + fmt.Printf("JSON list (%s) updated successfully\n", path) } diff --git a/flake.lock b/flake.lock index c723d287a..86f9ce4c7 100644 --- a/flake.lock +++ b/flake.lock @@ -22,6 +22,22 @@ "type": "github" } }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -74,11 +90,28 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1772736753, + "narHash": "sha256-au/m3+EuBLoSzWUCb64a/MZq6QUtOV8oC0D9tY2scPQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "917fec990948658ef1ccd07cef2a1ef060786846", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake-checks": "flake-checks", "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "tailscale-head": "tailscale-head" } }, "systems": { @@ -111,6 +144,41 @@ "type": "github" } }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "tailscale-head": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs_2", + "systems": "systems_3" + }, + "locked": { + "lastModified": 1782250635, + "narHash": "sha256-y4/EAApBPekjWz2cP/9RexTBkiSAXsdOHWnhm/no/4I=", + "owner": "tailscale", + "repo": "tailscale", + "rev": "d4f2917c1bfd27529ec7997d08a8f1aa9ea903f2", + "type": "github" + }, + "original": { + "owner": "tailscale", + "repo": "tailscale", + "type": "github" + } + }, "treefmt-nix": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index 96786def3..91cc9f8c8 100644 --- a/flake.nix +++ b/flake.nix @@ -9,10 +9,18 @@ # once it ships go_1_26 >= 1.26.4. nixpkgs.url = "github:NixOS/nixpkgs/staging-next-26.05"; flake-utils.url = "github:numtide/flake-utils"; + # Reusable Go flake checks (build/test/lint/format); CI runs them via # `nix build .#checks..` instead of bespoke per-tool steps. flake-checks.url = "github:kradalby/flake-checks"; flake-checks.inputs.nixpkgs.follows = "nixpkgs"; + + # Tailscale HEAD, built from source for the integration test client image. + # Bump with `nix flake update tailscale-head` to track the latest main. + # It is a flake; we still build our own derivation (we need derper + + # containerboot, which its default package omits) but read its committed + # vendorHash from flakehashes.json so the bump carries the hash. + tailscale-head.url = "github:tailscale/tailscale"; }; outputs = @@ -20,6 +28,7 @@ , nixpkgs , flake-utils , flake-checks + , tailscale-head , ... }: let @@ -39,8 +48,27 @@ # Go 1.26 builder; resolves to Go 1.26.4 from the pinned nixpkgs. buildGo = pkgs.buildGo126Module; vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri; + + # Go source with the non-Go scaffolding filtered out, so editing the + # nix integration harness, CI workflows, docs or packaging does not + # rebuild the (slow) integration test binary or container images. + # Only changes to actual Go source invalidate them. + goSrc = pkgs.lib.fileset.toSource { + root = ./.; + fileset = pkgs.lib.fileset.difference ./. (pkgs.lib.fileset.unions [ + ./nix + ./.github + ./docs + ./packaging + ./flake.nix + ./flake.lock + ./flakehashes.json + ]); + }; in { + headscale-go-src = goSrc; + headscale = buildGo { pname = "headscale"; version = headscaleVersion; @@ -71,6 +99,34 @@ subPackages = [ "cmd/hi" ]; }; + # The integration suite compiled into a single test binary, run later + # inside a per-test NixOS VM check (see nix/tests/integration.nix). + # Compiled once and shared by every integration check. + integration-test-bin = buildGo { + pname = "headscale-integration-test"; + # Fixed (not the commit rev) so the binary is content-addressed: + # identical Go source → identical store path → shared-cache hit + # across commits. The version string is irrelevant for a test binary. + version = "integration"; + src = goSrc; + + inherit vendorHash; + + doCheck = false; + + buildPhase = '' + runHook preBuild + go test -c -o integration.test ./integration + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + install -Dm755 integration.test $out/bin/integration.test + runHook postInstall + ''; + }; + # Build golangci-lint with stock Go 1.26 (upstream uses hardcoded Go # version); it does not build against the pinned 1.26.4. golangci-lint = buildGo rec { @@ -200,7 +256,12 @@ vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri; goPkg = pkgs.go_1_26; # //go:embed targets and test-read files outside the default whitelist. - embedDirs = [ ./hscontrol/assets ./hscontrol/db/schema.sql ./config-example.yaml ]; + embedDirs = [ + ./hscontrol/assets + ./hscontrol/db/schema.sql + ./config-example.yaml + ./integration/tailscale-versions.json + ]; extraSrc = [ ./hscontrol/testdata ./hscontrol/types/testdata @@ -239,6 +300,45 @@ fmtExclude = [ ./gen ./docs ]; }); }; + + # Nix-built container images for the integration tests, plus the + # per-test VM check factory. Linux-only (dockerTools + nixosTest). + integrationImages = import ./nix/images.nix { + inherit pkgs; + buildGoModule = pkgs.buildGo126Module; + tailscaleSrc = tailscale-head; + }; + + mkIntegrationCheck = { name, testFilter ? name, postgres ? false }: + pkgs.testers.nixosTest (import ./nix/tests/integration.nix { + inherit name testFilter pkgs postgres; + inherit (integrationImages) + headscaleImage tailscaleImage postgresImage tailscaleVersionImages; + testBin = pkgs.integration-test-bin; + # goSrc (not the full tree) so a check's result stays cached when + # only nix/CI/docs change — only real Go/integration changes (which + # also rebuild testBin) invalidate it. Maximises shared-cache reuse. + src = pkgs.headscale-go-src; + }); + + # One sqlite check per test (full matrix), plus a postgres variant for + # the postgres subset — exactly the sqlite+postgres jobs the generator + # produces for the GitHub workflow. + integrationChecks = + let + # Split-test entries (TestAutoApproveMultiNetwork/authkey-tag.*) carry + # / and * in their -test.run filter; sanitise those into a valid check + # name while keeping the raw filter for -test.run. + sanitize = builtins.replaceStrings [ "/" "." "*" ] [ "-" "" "" ]; + mkCheck = postgres: filter: + pkgs.lib.nameValuePair + "integration-${sanitize filter}${pkgs.lib.optionalString postgres "-pg"}" + (mkIntegrationCheck { name = sanitize filter; testFilter = filter; inherit postgres; }); + in + pkgs.lib.listToAttrs ( + map (mkCheck false) (import ./integration/tests.nix) + ++ map (mkCheck true) (import ./integration/postgres-tests.nix) + ); in { # `nix develop` @@ -259,6 +359,14 @@ cat go.mod | ${pkgs.ripgrep}/bin/rg "\t" | ${pkgs.ripgrep}/bin/rg -v indirect | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.findutils}/bin/xargs go get -u go mod tidy '') + + # Regenerate integration/tailscale-versions.json after a capver bump + # so the offline checks pin the tailscale images the suite requests. + # It is a //go:generate step in the integration package (runs with + # capver under `make generate`); this just scopes it to the pins. + (pkgs.writeShellScriptBin "update-integration-images" '' + exec go generate ./integration/... + '') ]; shellHook = '' @@ -272,6 +380,28 @@ inherit headscale; inherit headscale-docker; default = headscale; + } + // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { + headscale-integration-image = integrationImages.headscaleImage; + tailscale-integration-image = integrationImages.tailscaleImage; + postgres-integration-image = integrationImages.postgresImage; + inherit (pkgs) integration-test-bin; + + # All shared inputs every integration check pulls in: the test binary + # and every image. CI builds this once (a "prime" job) so it lands in + # the shared nix cache before the per-test matrix fans out — the + # matrix jobs then substitute it instead of each rebuilding. + integration-deps = pkgs.linkFarm "integration-deps" ( + [ + { name = "test-bin"; path = pkgs.integration-test-bin; } + { name = "headscale"; path = integrationImages.headscaleImage; } + { name = "tailscale-head"; path = integrationImages.tailscaleImage; } + { name = "postgres"; path = integrationImages.postgresImage; } + ] + ++ pkgs.lib.imap0 + (i: img: { name = "ts-version-${toString i}"; path = img; }) + integrationImages.tailscaleVersionImages + ); }; # `nix run` @@ -287,6 +417,10 @@ } # The Go build/test checks are gated to Linux: parts of the tree are # Linux-specific and the pure unit subset is validated by CI. - // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks; + // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks + # Integration checks only on x86_64-linux: nixosTest needs KVM and the + # pinned version images are amd64 (nix/tailscale-versions.nix). aarch64 + # would need arch-specific pins. + // pkgs.lib.optionalAttrs (system == "x86_64-linux") integrationChecks; }); } diff --git a/integration/excluded-tests.json b/integration/excluded-tests.json new file mode 100644 index 000000000..a9160c4da --- /dev/null +++ b/integration/excluded-tests.json @@ -0,0 +1,7 @@ +[ + "TestHASubnetRouterFailoverBothOfflineCablePull", + "TestHASubnetRouterFailoverDockerDisconnect", + "TestHASubnetRouterPingFailover", + "TestPingAllByIPPublicDERP", + "TestTailscaleRustAxum" +] diff --git a/integration/postgres-tests.nix b/integration/postgres-tests.nix new file mode 100644 index 000000000..68d9a2228 --- /dev/null +++ b/integration/postgres-tests.nix @@ -0,0 +1,8 @@ +# Generated by gh-action-integration-generator.go; do not edit. +[ + "TestACLAllowUserDst" + "TestPingAllByIP" + "TestEphemeral2006DeletedTooQuickly" + "TestPingAllByIPManyUpDown" + "TestSubnetRouterMultiNetwork" +] diff --git a/integration/tests.nix b/integration/tests.nix new file mode 100644 index 000000000..f2f2d1675 --- /dev/null +++ b/integration/tests.nix @@ -0,0 +1,156 @@ +# Generated by gh-action-integration-generator.go; do not edit. +[ + "TestACLHostsInNetMapTable" + "TestACLAllowUser80Dst" + "TestACLDenyAllPort80" + "TestACLAllowUserDst" + "TestACLAllowStarDst" + "TestACLNamedHostsCanReachBySubnet" + "TestACLNamedHostsCanReach" + "TestACLDevice1CanAccessDevice2" + "TestPolicyUpdateWhileRunningWithCLIInDatabase" + "TestACLAutogroupMember" + "TestACLAutogroupTagged" + "TestACLAutogroupSelf" + "TestACLPolicyPropagationOverTime" + "TestACLTagPropagation" + "TestACLTagPropagationPortSpecific" + "TestACLGroupWithUnknownUser" + "TestACLGroupAfterUserDeletion" + "TestACLGroupDeletionExactReproduction" + "TestACLDynamicUnknownUserAddition" + "TestACLDynamicUnknownUserRemoval" + "TestAPIAuthenticationBypass" + "TestAPIAuthenticationBypassCurl" + "TestRemoteCLIAuthenticationBypass" + "TestCLIWithConfigAuthenticationBypass" + "TestAuthKeyLogoutAndReloginSameUser" + "TestAuthKeyLogoutAndReloginNewUser" + "TestAuthKeyLogoutAndReloginSameUserExpiredKey" + "TestAuthKeyDeleteKey" + "TestAuthKeyLogoutAndReloginRoutesPreserved" + "TestOIDCAuthenticationPingAll" + "TestOIDCExpireNodesBasedOnTokenExpiry" + "TestOIDC024UserCreation" + "TestOIDCAuthenticationWithPKCE" + "TestOIDCReloginSameNodeNewUser" + "TestOIDCFollowUpUrl" + "TestOIDCMultipleOpenedLoginUrls" + "TestOIDCReloginSameNodeSameUser" + "TestOIDCExpiryAfterRestart" + "TestOIDCACLPolicyOnJoin" + "TestOIDCReloginSameUserRoutesPreserved" + "TestAuthWebFlowAuthenticationPingAll" + "TestAuthWebFlowLogoutAndReloginSameUser" + "TestAuthWebFlowLogoutAndReloginNewUser" + "TestApiKeyCommand" + "TestApiKeyCommandValidation" + "TestAuthCommandValidation" + "TestNodeCommand" + "TestNodeExpireCommand" + "TestNodeRenameCommand" + "TestPreAuthKeyCorrectUserLoggedInCommand" + "TestTaggedNodesCLIOutput" + "TestNodeExpireFlagsCommand" + "TestNodeCommandValidation" + "TestNodeTagCommand" + "TestNodeRouteCommands" + "TestNodeBackfillIPsCommand" + "TestPolicyCheckCommand" + "TestSSHTestsRejectFailingPolicy" + "TestPolicyCommand" + "TestPolicyBrokenConfigCommand" + "TestPreAuthKeyCommand" + "TestPreAuthKeyCommandWithoutExpiry" + "TestPreAuthKeyCommandReusableEphemeral" + "TestPreAuthKeyDeleteCommand" + "TestPreAuthKeyCommandValidation" + "TestServerInfoCommands" + "TestUserCommand" + "TestUserCreateCommand" + "TestUserCommandValidation" + "TestDERPVerifyEndpoint" + "TestResolveMagicDNS" + "TestResolveMagicDNSExtraRecordsPath" + "TestDERPServerScenario" + "TestDERPServerWebsocketScenario" + "TestPingAllByIP" + "TestEphemeral" + "TestEphemeralInAlternateTimezone" + "TestEphemeral2006DeletedTooQuickly" + "TestPingAllByHostname" + "TestTaildrop" + "TestUpdateHostnameFromClient" + "TestExpireNode" + "TestSetNodeExpiryInFuture" + "TestDisableNodeExpiry" + "TestNodeOnlineStatus" + "TestPingAllByIPManyUpDown" + "Test2118DeletingOnlineNodePanics" + "TestGrantCapRelay" + "TestGrantCapDrive" + "TestEnablingRoutes" + "TestHASubnetRouterFailover" + "TestSubnetRouteACL" + "TestEnablingExitRoutes" + "TestExitRoutesWithAutogroupInternetACL" + "TestSubnetRouterMultiNetwork" + "TestSubnetRouterMultiNetworkExitNode" + "TestAutoApproveMultiNetwork/authkey-tag.*" + "TestAutoApproveMultiNetwork/authkey-user.*" + "TestAutoApproveMultiNetwork/authkey-group.*" + "TestAutoApproveMultiNetwork/webauth-tag.*" + "TestAutoApproveMultiNetwork/webauth-user.*" + "TestAutoApproveMultiNetwork/webauth-group.*" + "TestSubnetRouteACLFiltering" + "TestGrantViaSubnetSteering" + "TestHASubnetRouterFailoverBothOffline" + "TestHeadscale" + "TestTailscaleNodesJoiningHeadcale" + "TestSSHOneUserToAll" + "TestSSHMultipleUsersAllToAll" + "TestSSHNoSSHConfigured" + "TestSSHIsBlockedInACL" + "TestSSHUserOnlyIsolation" + "TestSSHAutogroupSelf" + "TestSSHOneUserToOneCheckModeCLI" + "TestSSHOneUserToOneCheckModeOIDC" + "TestSSHCheckModeUnapprovedTimeout" + "TestSSHCheckModeCheckPeriodCLI" + "TestSSHCheckModeAutoApprove" + "TestSSHCheckModeSessionLossReDelegates" + "TestSSHCheckModeNegativeCLI" + "TestSSHLocalpart" + "TestTagsAuthKeyWithTagRequestDifferentTag" + "TestTagsAuthKeyWithTagNoAdvertiseFlag" + "TestTagsAuthKeyWithTagCannotAddViaCLI" + "TestTagsAuthKeyWithTagCannotChangeViaCLI" + "TestTagsAuthKeyWithTagAdminOverrideReauthPreserves" + "TestTagsAuthKeyWithTagCLICannotModifyAdminTags" + "TestTagsAuthKeyWithoutTagCannotRequestTags" + "TestTagsAuthKeyWithoutTagRegisterNoTags" + "TestTagsAuthKeyWithoutTagCannotAddViaCLI" + "TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset" + "TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise" + "TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag" + "TestTagsUserLoginOwnedTagAtRegistration" + "TestTagsUserLoginNonExistentTagAtRegistration" + "TestTagsUserLoginUnownedTagAtRegistration" + "TestTagsUserLoginAddTagViaCLIReauth" + "TestTagsUserLoginRemoveTagViaCLIReauth" + "TestTagsUserLoginCLINoOpAfterAdminAssignment" + "TestTagsUserLoginCLICannotRemoveAdminTags" + "TestTagsAuthKeyWithTagRequestNonExistentTag" + "TestTagsAuthKeyWithTagRequestUnownedTag" + "TestTagsAuthKeyWithoutTagRequestNonExistentTag" + "TestTagsAuthKeyWithoutTagRequestUnownedTag" + "TestTagsAdminAPICannotSetNonExistentTag" + "TestTagsAdminAPICanSetUnownedTag" + "TestTagsAdminAPICannotRemoveAllTags" + "TestTagsIssue2978ReproTagReplacement" + "TestTagsAdminAPICannotSetInvalidFormat" + "TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags" + "TestTagsAuthKeyWithoutUserInheritsTags" + "TestTagsAuthKeyWithoutUserRejectsAdvertisedTags" + "TestTagsAuthKeyConvertToUserViaCLIRegister" +] diff --git a/nix/tests/integration.nix b/nix/tests/integration.nix new file mode 100644 index 000000000..e96a861d9 --- /dev/null +++ b/nix/tests/integration.nix @@ -0,0 +1,157 @@ +# Per-test integration check: a NixOS VM that boots docker, loads the nix-built +# + pinned images (full tailscale version matrix), and runs the prebuilt +# integration test binary filtered to a single test. Hermetic equivalent of one +# `go run ./cmd/hi run ` job. `postgres = true` produces the postgres +# variant (HEADSCALE_INTEGRATION_POSTGRES=1) for the postgres subset. +# +# The suite runs *inside* a container named headscale-test-suite- with +# the docker socket mounted (docker-out-of-docker): scenario.go attaches that +# container to each test network, and IntegrationSkip gates on /.dockerenv. We +# reuse the headscale image as the runner base and bind-mount the host nix store +# so the test binary's closure resolves. +{ name +, testFilter ? name +, pkgs +, headscaleImage +, tailscaleImage +, postgresImage +, tailscaleVersionImages +, testBin +, src +, postgres ? false +}: +let + lib = pkgs.lib; + + # The test function name, dropping any "/subtest" filter on a split check, so + # the skip-guard below matches a whole-function skip without tripping on a + # legitimately-skipped subtest. + testFunc = builtins.head (lib.splitString "/" testFilter); + + # >=6 chars and DNS-safe: tsic/hsic derive container hostnames from the last + # 6 chars of the run ID (runID[len-6:]), which panics on shorter ids. + runID = "nixcheck"; + runnerName = "headscale-test-suite-${runID}"; + + # Images this check must have locally before the suite runs. Every test may + # fan out over the full tailscale version matrix (MustTestVersions); the head + # image doubles as the DERP server; the postgres variant adds postgres. + images = + [ headscaleImage tailscaleImage ] + ++ tailscaleVersionImages + ++ lib.optional postgres postgresImage; + + # Loading ~8 images dominates wall time; do it in parallel (they share layers, + # so docker dedups) and fail if any load fails. + loadImages = pkgs.writeShellScript "load-integration-images" '' + set -u + pids="" + ${lib.concatMapStringsSep "\n" (img: ''docker load -i ${img} & pids="$pids $!"'') images} + rc=0 + for p in $pids; do wait "$p" || rc=1; done + exit "$rc" + ''; + + pgEnv = lib.optionalString postgres + "-e HEADSCALE_INTEGRATION_POSTGRES=1 -e HEADSCALE_INTEGRATION_POSTGRES_IMAGE=postgres:16 "; +in +{ + name = "integration-${name}${lib.optionalString postgres "-pg"}"; + + nodes.machine = { ... }: { + virtualisation.docker.enable = true; + + # Match the integration CI: Docker 29's default containerd snapshotter + + # overlayfs regresses image load/exec for these images; the classic + # overlay2 graph driver is the known-good path. + virtualisation.docker.daemon.settings = { + storage-driver = "overlay2"; + features.containerd-snapshotter = false; + }; + + # tailscaled needs /dev/net/tun for its --tun=tsdev device. + boot.kernelModules = [ "tun" ]; + + # The VM's dhcpcd otherwise manages docker's bridges/veths: as containers + # and the cable-pull tests churn interfaces it deletes their routes and + # races libnetwork's gateway setup, surfacing as + # "API error (500): updating gateway endpoint: failed to set gateway: + # network is unreachable" on reconnect (TestHASubnetRouterFailover*Disconnect + # /CablePull) and intermittent container-startup flakes. Keep it off them. + networking.dhcpcd.denyInterfaces = [ "veth*" "br-*" "docker*" ]; + + # Trim the guest toward a tiny profile: this VM only needs to boot, run + # docker, and exec one test binary. Measured boot is 18.7s (1.7s kernel + + # 3.9s initrd + 13.1s userspace); the userspace critical chain is + # dhcpcd (6.1s) -> network-online.target -> docker.service (+3.5s), i.e. the + # remaining cost is functional (docker waits for the network), not fat to + # trim. documentation + timesyncd off shave the cheap, avoidable part. + # + # microvm.nix was evaluated against this and rejected: it still needs a full + # docker daemon + nested KVM, and boot is only ~10-15% of the 2-3 min + # image-load + test runtime, so a new flake input can't pay itself back. The + # one real lever left, dropping docker's network-online wait (~6s), sits + # right next to the dhcpcd interface races handled below and isn't worth + # destabilising the matrix for. + documentation.enable = false; + services.timesyncd.enable = false; + + # Sized for the full version matrix (a test can spawn 12+ tailscale + # containers + headscale + the runner) while still fitting a standard + # 4-core/16 GB GitHub runner. Disk-backed docker (not tmpfs) keeps RAM + # headroom on the runner; the slim source copy below is the bigger win. + virtualisation.memorySize = 8192; + virtualisation.cores = 4; + virtualisation.diskSize = 12288; + }; + + testScript = '' + start_all() + machine.wait_for_unit("docker.service") + + # Load all matrix images in parallel (store paths visible inside the VM). + machine.succeed("${loadImages}", timeout=600) + + # Writable copy of just the integration package (the suite writes + # control_logs/ under CWD and reads testdata relative to it). Only this + # subtree is needed at runtime — the repo-root build context is only used + # for image builds, which we bypass with prebuilt images. + machine.succeed("mkdir -p /tmp/hs && cp -r ${src}/integration /tmp/hs/integration && chmod -R u+w /tmp/hs") + + # hsic/tsic save artifacts (and tsic.Status writes _status.json) under + # /tmp/control; cmd/hi mounts this dir. Without it every Status() errors on + # the write and the suite never observes NeedsLogin. + machine.succeed("mkdir -p /tmp/control") + + # Run the suite inside a container (docker-out-of-docker), mirroring cmd/hi: + # - name matches scenario.go's testSuiteName so it can join test networks + # - docker socket mounted so sibling containers spawn on this daemon + # - host nix store mounted so the test binary's closure resolves + out = machine.succeed( + "docker run --rm --name ${runnerName} " + "-v /var/run/docker.sock:/var/run/docker.sock " + "-v /nix/store:/nix/store:ro " + "-v /tmp/hs:/tmp/hs -w /tmp/hs/integration " + "-v /tmp/control:/tmp/control " + "-e HEADSCALE_INTEGRATION_HEADSCALE_IMAGE=headscale:nix " + "-e HEADSCALE_INTEGRATION_TAILSCALE_IMAGE=tailscale-head:nix " + "-e HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE=tailscale-head:nix " + "-e HEADSCALE_INTEGRATION_DERPER_IMAGE=tailscale-head:nix " + "${pgEnv}" + "-e HEADSCALE_INTEGRATION_RUN_ID=${runID} " + "-e CI=1 " + # The nested-docker VM converges slower than a host docker daemon; widen + # the ScaledTimeout budgets (subnet-route convergence, HA cycles) past 2x. + "-e HEADSCALE_INTEGRATION_TIMEOUT_SCALE=4 " + "-e SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt " + "headscale:nix " + "${testBin}/bin/integration.test -test.run '^${testFilter}$' -test.v -test.timeout=20m", + timeout=1800, + ) + print(out) + + # A skipped or unmatched test still exits 0; guard against a false green. + assert "no tests to run" not in out, "filter ${testFilter} matched nothing" + assert "--- SKIP: ${testFunc} " not in out, "test ${testFunc} was skipped (not in container?)" + ''; +}