From df6bb833dd44e4d2f92462ecd36d6d9efd385eea Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 25 Jun 2026 08:05:46 +0000 Subject: [PATCH 1/7] ci: pin the integration tailscale versions, generated from capver The hermetic nix checks cannot pull tailscale images at eval time, so the tested version set and the per-arch image digests are pinned in integration/tailscale-versions.json. capver.IntegrationVersions() is the single source the suite, the drift guard and the pin generator share; the generator (tools/tailscale-versions, run via go generate) refreshes it after a capver bump and reuses already-present pins so a routine generate makes no network call. --- .prettierignore | 4 + cmd/hi/listversions.go | 4 +- hscontrol/capver/capver.go | 14 ++ integration/tailscale-versions.json | 66 +++++++++ tools/tailscale-versions/main.go | 217 ++++++++++++++++++++++++++++ 5 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 integration/tailscale-versions.json create mode 100644 tools/tailscale-versions/main.go diff --git a/.prettierignore b/.prettierignore index 9d8aa3bc9..a8dd52973 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1,6 @@ .github/workflows/test-integration-v2* docs/ +# Generated by gh-action-integration-generator.go (its own canonical format). +integration/excluded-tests.json +# Generated by update-integration-images (jq output). +integration/tailscale-versions.json diff --git a/cmd/hi/listversions.go b/cmd/hi/listversions.go index c2162ad64..ece773c1d 100644 --- a/cmd/hi/listversions.go +++ b/cmd/hi/listversions.go @@ -30,9 +30,7 @@ var listVersionsConfig ListVersionsConfig // tags, releases get a "v" prefix so each entry can be appended to // "ghcr.io/tailscale/tailscale:" directly. func listVersions(env *command.Env) error { - release := capver.TailscaleLatestMajorMinor(capver.SupportedMajorMinorVersions, true) - all := append([]string{"head", "unstable"}, release...) - must := append(append([]string{}, all[0:4]...), all[len(all)-2:]...) + all, must := capver.IntegrationVersions() var versions []string diff --git a/hscontrol/capver/capver.go b/hscontrol/capver/capver.go index a4c535e88..5ebe2ce85 100644 --- a/hscontrol/capver/capver.go +++ b/hscontrol/capver/capver.go @@ -84,3 +84,17 @@ func TailscaleLatestMajorMinor(n int, stripV bool) []string { return majorSl[len(majorSl)-n:] } + +// IntegrationVersions returns the bare Tailscale version tags the integration +// suite pins and tests — the full set and the "must" subset (head, unstable, +// the two newest and two oldest tracked releases, for boundary coverage). +// Releases are bare (1.80); callers that build image tags prepend "v". This is +// the one source the suite's version list, the drift guard (`hi list-versions`), +// and the tailscale-versions.json generator all derive from. +func IntegrationVersions() ([]string, []string) { + release := TailscaleLatestMajorMinor(SupportedMajorMinorVersions, true) + all := append([]string{"head", "unstable"}, release...) + must := append(append([]string{}, all[0:4]...), all[len(all)-2:]...) + + return all, must +} diff --git a/integration/tailscale-versions.json b/integration/tailscale-versions.json new file mode 100644 index 000000000..ca2b7a346 --- /dev/null +++ b/integration/tailscale-versions.json @@ -0,0 +1,66 @@ +{ + "versions": [ + "head", + "unstable", + "1.80", + "1.82", + "1.96", + "1.98" + ], + "images": { + "unstable": { + "imageName": "tailscale/tailscale", + "finalImageTag": "unstable", + "imageDigest": "sha256:85d46a2b396edbc3c0b1cda1d4433831236aa64da32fc80af0897501b1ab9ae2", + "sha256": { + "amd64": "sha256-UYtRaPnnDgRNomEX5EGp6yf/ye1KB/5RwaU2KWNRLsU=", + "arm64": "sha256-OmK7uLVskXSgto+ywvGlnhG5AhXOhlRYz4hi2dvp/a4=" + } + }, + "v1.80": { + "imageName": "ghcr.io/tailscale/tailscale", + "finalImageTag": "v1.80", + "imageDigest": "sha256:27b6a3dc30d89e94113b0d481dae05f08934cf80bdce860041727a2a60959921", + "sha256": { + "amd64": "sha256-CGNdS2gdO7VSG1NNOBUTxF4oeIwnUO00AWxyzwhWshs=", + "arm64": "sha256-YgX5Yy2Ls7HKOAXhwNNOQ9fgzK9EZqiRc9Q1Vej9A38=" + } + }, + "v1.82": { + "imageName": "ghcr.io/tailscale/tailscale", + "finalImageTag": "v1.82", + "imageDigest": "sha256:d26fc9bb035b0559900cc6f23506f6b1ddab61a690ffab4f5d84feceb3de811e", + "sha256": { + "amd64": "sha256-46PK+HTq6WVawxXj9Pktc1zHHGce/cSNAKAWFNX5NYA=", + "arm64": "sha256-Ij6qPQgECR7AGti8NCoNg5VaUZq3dKjQreQTOz/dDHE=" + } + }, + "v1.96": { + "imageName": "ghcr.io/tailscale/tailscale", + "finalImageTag": "v1.96", + "imageDigest": "sha256:dbeff02d2337344b351afac203427218c4d0a06c43fc10a865184063498472a6", + "sha256": { + "amd64": "sha256-F2vR1IhVeBOVRILxWIQ1N8DCX6foWmp6B/BZbFRgf5Q=", + "arm64": "sha256-q+hCmoH7BIX1B4BPBC9HJRlB7KHxFH9fFB6RKCYeK3A=" + } + }, + "v1.98": { + "imageName": "ghcr.io/tailscale/tailscale", + "finalImageTag": "v1.98", + "imageDigest": "sha256:854b77123b9536adae2e97f5a5fdb1790ed03438b911ab7f07780155e0af6ce2", + "sha256": { + "amd64": "sha256-zmPhOuNhcnExtY8L1G2za28fHpltgcmcdvGSEWcPVaY=", + "arm64": "sha256-J37dagB9SFlhAPTWQShty6muTmU71KLFKhBwpUtnxHc=" + } + } + }, + "postgres": { + "imageName": "postgres", + "finalImageTag": "16", + "imageDigest": "sha256:287eced1f33b59ed265ed13a60d3680dd7646d70c4dc0e785f59a470ebc03eeb", + "sha256": { + "amd64": "sha256-NgSEkOyXG8tdX9qRInuTd0n0fbVDKlPkJZz7haqFNMo=", + "arm64": "sha256-gcIWSO7wg1HlRUafEN7X9KdqpMgnar6gRf5IT7BiOdY=" + } + } +} diff --git a/tools/tailscale-versions/main.go b/tools/tailscale-versions/main.go new file mode 100644 index 000000000..814bd6bd0 --- /dev/null +++ b/tools/tailscale-versions/main.go @@ -0,0 +1,217 @@ +// Command tailscale-versions regenerates integration/tailscale-versions.json: +// the bare version list the suite tests (from capver) plus the per-arch +// registry pins (manifest digest + nix sha256) the offline nix checks load. +// +// It is wired as a //go:generate step in the integration package so a capver +// bump refreshes the pins in the same `make generate` pass as capver itself. It +// shells out to nix-prefetch-docker, so it runs in the nix dev shell — but only +// for tags that aren't already pinned: a routine generate with an unchanged +// version set makes no network call and rewrites nothing. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + + "github.com/juanfont/headscale/hscontrol/capver" +) + +// versionsJSONPath is relative to the integration package directory, which is +// the working directory go:generate runs this command from. +const versionsJSONPath = "tailscale-versions.json" + +// genArches are the docker architectures pinned for every image. The registry +// images are multi-arch, so the imageDigest (the manifest index) is shared and +// only the nix sha256 differs per arch; pullImage selects by the build system. +var genArches = []string{"amd64", "arm64"} + +var ( + reImageDigest = regexp.MustCompile(`imageDigest = "([^"]+)"`) + reHash = regexp.MustCompile(`hash = "([^"]+)"`) + + errPrefetchEmpty = errors.New("nix-prefetch-docker produced no digest/hash (rate limited? retry)") +) + +// imageRef pins a multi-arch registry image for dockerTools.pullImage. +type imageRef struct { + ImageName string `json:"imageName"` + FinalImageTag string `json:"finalImageTag"` + ImageDigest string `json:"imageDigest"` + SHA256 map[string]string `json:"sha256"` +} + +// complete reports whether the pin already has a digest and every arch hash, so +// it can be reused without re-hitting the registry. +func (r imageRef) complete() bool { + if r.ImageDigest == "" { + return false + } + + for _, a := range genArches { + if r.SHA256[a] == "" { + return false + } + } + + return true +} + +// versionsFile is the single source of truth shared by the Go suite +// (MustTestVersions reads .versions) and the nix image pins (images.nix reads +// .images / .postgres). See integration/tailscale-versions.json. +type versionsFile struct { + Versions []string `json:"versions"` + Images map[string]imageRef `json:"images"` + Postgres imageRef `json:"postgres"` +} + +func main() { + err := run() + if err != nil { + fmt.Fprintln(os.Stderr, "tailscale-versions:", err) + os.Exit(1) + } +} + +func run() error { + _, must := capver.IntegrationVersions() + + prev := readExisting() + + out := versionsFile{Versions: must, Images: map[string]imageRef{}} + + for _, v := range must { + // head is built from source, not pulled. + if v == "head" { + continue + } + + repo, tag := "ghcr.io/tailscale/tailscale", "v"+v + if v == "unstable" { + repo, tag = "tailscale/tailscale", "unstable" + } + + ref, err := pinOrReuse(prev.Images[tag], repo, tag) + if err != nil { + return err + } + + out.Images[tag] = ref + } + + postgres, err := pinOrReuse(prev.Postgres, "postgres", "16") + if err != nil { + return err + } + + out.Postgres = postgres + + return writeIfChanged(out) +} + +// pinOrReuse keeps an already-complete pin so a routine generate makes no +// network call; only new or incomplete tags are prefetched. To deliberately +// refresh a mutable tag (unstable, postgres) delete its entry first. +func pinOrReuse(prev imageRef, repo, tag string) (imageRef, error) { + if prev.complete() { + return prev, nil + } + + return pinImage(context.Background(), repo, tag) +} + +// readExisting loads the current pins, or an empty file if absent/unparseable +// (a full regeneration). +func readExisting() versionsFile { + vf := versionsFile{Images: map[string]imageRef{}} + + b, err := os.ReadFile(versionsJSONPath) + if err != nil { + return vf + } + + err = json.Unmarshal(b, &vf) + if err != nil { + return versionsFile{Images: map[string]imageRef{}} + } + + if vf.Images == nil { + vf.Images = map[string]imageRef{} + } + + return vf +} + +// writeIfChanged only touches the file when the rendered JSON differs, so a +// no-op generate leaves the working tree (and git) clean. +func writeIfChanged(out versionsFile) error { + b, err := json.MarshalIndent(out, "", " ") + if err != nil { + return fmt.Errorf("marshalling versions: %w", err) + } + + b = append(b, '\n') + + old, err := os.ReadFile(versionsJSONPath) + if err == nil && bytes.Equal(old, b) { + fmt.Printf("%s already up to date\n", versionsJSONPath) + + return nil + } + + err = os.WriteFile(versionsJSONPath, b, 0o600) + if err != nil { + return fmt.Errorf("writing %s: %w", versionsJSONPath, err) + } + + fmt.Printf("wrote %s\n", versionsJSONPath) + + return nil +} + +// pinImage prefetches repo:tag for every arch via nix-prefetch-docker and +// returns the shared manifest digest plus per-arch nix hashes. +func pinImage(ctx context.Context, repo, tag string) (imageRef, error) { + ref := imageRef{ImageName: repo, FinalImageTag: tag, SHA256: map[string]string{}} + + for _, arch := range genArches { + cmd := exec.CommandContext(ctx, "nix", "run", "nixpkgs#nix-prefetch-docker", "--", + "--image-name", repo, "--image-tag", tag, "--arch", arch, "--os", "linux") + + var stdout bytes.Buffer + + cmd.Stdout = &stdout + + err := cmd.Run() + if err != nil { + return imageRef{}, fmt.Errorf("prefetch %s:%s (%s): %w", repo, tag, arch, err) + } + + digest := firstSubmatch(reImageDigest, stdout.String()) + hash := firstSubmatch(reHash, stdout.String()) + + if digest == "" || hash == "" { + return imageRef{}, fmt.Errorf("%s:%s (%s): %w", repo, tag, arch, errPrefetchEmpty) + } + + ref.ImageDigest = digest + ref.SHA256[arch] = hash + } + + return ref, nil +} + +func firstSubmatch(re *regexp.Regexp, s string) string { + m := re.FindStringSubmatch(s) + if len(m) < 2 { + return "" + } + + return m[1] +} From a5152db397b942076156602b886662b2a68d52e9 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 25 Jun 2026 08:06:34 +0000 Subject: [PATCH 2/7] nix: build the headscale and tailscale-HEAD test images Replaces the Dockerfile builds for the flake-check path. headscale is built from the Go-filtered source; one tailscale-HEAD image serves client, daemon, derper and the websocket-DERP test. Released tailscale versions and postgres are pinned registry pulls, arch following the build system. Tools the suite used to apk-add (curl/ssh/dig/python3) and the SSH login users are baked in at build time so no container is mutated at startup, and busybox su is included so taildrive can fork its per-user fileserver. --- nix/images.nix | 236 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 nix/images.nix diff --git a/nix/images.nix b/nix/images.nix new file mode 100644 index 000000000..1a91453cb --- /dev/null +++ b/nix/images.nix @@ -0,0 +1,236 @@ +# Nix-built + pinned container images for the integration tests. +# +# Built images (headscale, tailscale-HEAD, derper) replace +# Dockerfile.integration-ci / Dockerfile.tailscale-HEAD / Dockerfile.derper for +# the nix/flake-check path. Released tailscale versions + postgres are pinned +# pulls (nix/tailscale-versions.nix). Everything is loaded into the VM docker so +# the checks run the full version matrix offline; the Go test code is unchanged +# and consumes them via the same HEADSCALE_INTEGRATION_*_IMAGE env vars / repo +# tags the `hi` CI path already uses. +{ pkgs, buildGoModule, tailscaleSrc }: +let + lib = pkgs.lib; + + # Single source of truth for the tailscale version matrix + pinned registry + # images, shared with the Go suite (MustTestVersions reads the same file). + # Regenerate with `update-integration-images`. + versions = builtins.fromJSON (builtins.readFile ../integration/tailscale-versions.json); + + # amd64/arm64 follows the build system; the tailscale + postgres registry + # images are multi-arch so the imageDigest (the index) is shared and we select + # the per-arch nix hash. Run on whatever arch the runner is, no forced cross-build. + dockerArch = if pkgs.stdenv.hostPlatform.isAarch64 then "arm64" else "amd64"; + + # Same headscale build, but from the Go-only filtered source and a fixed + # version (not the commit rev) so the image is content-addressed: identical + # source → identical store path → shared-cache hit across commits. + headscale = pkgs.headscale.overrideAttrs (_: { + src = pkgs.headscale-go-src; + version = "integration"; + }); + + # alpine's `update-ca-certificates` has no drop-in nixpkgs equivalent. The + # integration containers write the per-test headscale CA to + # /usr/local/share/ca-certificates/user-N.crt and then call + # update-ca-certificates so TLS to headscale is trusted (integration/tsic, + # integration/hsic). This shim reproduces exactly that: append those certs + # onto a writable bundle that Go reads via SSL_CERT_FILE. + # + # ponytail: minimal CA shim; swap for a real ca-certificates package only if a + # test needs the full openssl rehash dance. + caShim = pkgs.writeShellScriptBin "update-ca-certificates" '' + set -eu + bundle=/etc/ssl/certs/ca-certificates.crt + mkdir -p /etc/ssl/certs + if [ ! -s "$bundle" ]; then + cp ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt "$bundle" + fi + for f in /usr/local/share/ca-certificates/*.crt; do + [ -e "$f" ] || continue + cat "$f" >> "$bundle" + done + ''; + + # taildrive (TestGrantCapDrive) spawns its per-user fileserver by re-execing + # tailscaled under `su` (tailscale drive/driveimpl/remote_impl.go): tailscaled + # runs as root and sus to the share's user. With no `su` on PATH canSU() fails, + # the fallback hits assertNotRoot() while root, the fileserver never starts, + # and every share fails with "unable to determine address for share". The + # released alpine images ship busybox su; expose just that applet here so it + # does not shadow coreutils on the image PATH. + busyboxSu = pkgs.runCommand "busybox-su" { } '' + mkdir -p $out/bin + ln -s ${pkgs.busybox}/bin/busybox $out/bin/su + ''; + + # Tailscale HEAD client + daemon + derper, built once from the pinned source. + # Mirrors Dockerfile.tailscale-HEAD / Dockerfile.derper's `go install`. + tailscaleHead = buildGoModule { + pname = "tailscale-head"; + version = "head"; + src = tailscaleSrc; + + # Tailscale commits its own Go module vendorHash to flakehashes.json; reuse + # it so `nix flake update tailscale-head` carries the matching hash and we + # never hand-maintain it. (The hash tracks go.mod, not our subPackages.) + vendorHash = (builtins.fromJSON (builtins.readFile "${tailscaleSrc}/flakehashes.json")).vendor.sri; + + subPackages = [ "cmd/tailscale" "cmd/tailscaled" "cmd/containerboot" "cmd/derper" ]; + + # ts_debug_websockets so this one image also serves the websocket-DERP test + # (no separate build); the code path is inert without TS_DEBUG_DERP_WS_CLIENT. + tags = [ "ts_debug_websockets" ]; + + env.CGO_ENABLED = "0"; + doCheck = false; + }; + + # Userland the integration suite shells out to inside the containers: + # sh/sleep/cat (coreutils), find (findutils), ps/pidof (procps), + # grep/sed/awk, bash. Missing any of these makes test Execute() calls fail. + baseUserland = with pkgs; [ + bashInteractive + coreutils + findutils + procps + gnugrep + gnused + gawk + ]; + + # Shared image scaffolding. Go reads /etc/ssl/certs/ca-certificates.crt via + # SSL_CERT_FILE (maintained by caShim at container start). + commonEnv = [ + "PATH=/bin:/usr/bin:/usr/local/bin:/sbin" + "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt" + ]; + # /var/lib/tailscale must exist and be writable, else tailscaled's default + # state path resolves empty and it exits with "--state is required" + # (tailscale paths/paths_unix.go stateFileUnix). derper-certs holds the + # manual cert dsic writes before starting derper. + mkWritableDirs = '' + mkdir -p etc/ssl/certs usr/local/share/ca-certificates usr/local/share/derper-certs etc/headscale tmp var/run var/lib/tailscale + ''; + + # Local login accounts the SSH integration tests map tailscale SSH sessions to. + # Pre-created in both images at build time (head: seeded passwd below; released: + # busybox adduser in withTestTools) so no container is mutated at startup. + sshTestUsers = [ "ssh-it-user" "user1" "user2" ]; + passwdFile = pkgs.writeText "passwd" ('' + root:x:0:0:root:/root:/bin/sh + nobody:x:65534:65534:nobody:/:/bin/sh + '' + lib.concatStrings (lib.imap0 + (i: u: "${u}:x:${toString (1000 + i)}:${toString (1000 + i)}::/home/${u}:/bin/sh\n") + sshTestUsers)); + groupFile = pkgs.writeText "group" ('' + root:x:0: + nobody:x:65534: + '' + lib.concatStrings (lib.imap0 (i: u: "${u}:x:${toString (1000 + i)}:\n") sshTestUsers)); + + tailscaleImage = pkgs.dockerTools.buildLayeredImage { + name = "tailscale-head"; + tag = "nix"; + contents = [ + tailscaleHead + caShim + pkgs.dockerTools.binSh # /bin/sh -> bash (tsic entrypoint is /bin/sh -c) + pkgs.iproute2 # ip + pkgs.iptables # iptables, ip6tables + pkgs.curl + pkgs.python3 # webserver tests + pkgs.dnsutils # dig/nslookup (dns tests apk-add bind-tools) + pkgs.openssh # ssh, for the SSH tests + pkgs.getent # tailscale SSH server execs `getent passwd ` + pkgs.hostname # SSH tests run `hostname` on the peer over the session + pkgs.traceroute # HA/route tests assert path via traceroute (head-only) + busyboxSu # taildrive sus to spawn its per-user fileserver (TestGrantCapDrive) + ] ++ baseUserland; + # buildLayeredImage puts every package's bin on /bin, which is on PATH, so + # the suite resolves tailscale/tailscaled/derper/curl/ssh/... directly — no + # symlinks needed. Just the writable state dirs and a passwd with the login + # the SSH tests use (ssh-it-user, pre-created so no runtime adduser). + extraCommands = '' + ${mkWritableDirs} + ${lib.concatMapStringsSep "\n" (u: "mkdir -p home/${u}") sshTestUsers} + cp ${passwdFile} etc/passwd + cp ${groupFile} etc/group + ''; + config.Env = commonEnv; + }; + + headscaleImage = pkgs.dockerTools.buildLayeredImage { + name = "headscale"; + tag = "nix"; + contents = [ + headscale + caShim + pkgs.dockerTools.binSh # hsic entrypoint is /bin/bash -c + pkgs.iproute2 # ip + pkgs.jq + pkgs.sqlite + pkgs.python3 + pkgs.curl + pkgs.dnsutils # dig + ] ++ baseUserland; + extraCommands = '' + ${mkWritableDirs} + mkdir -p usr/local/bin + ln -sf ${headscale}/bin/headscale usr/local/bin/headscale + ''; + config.Env = commonEnv; + }; + + # The tools the suite used to `apk add` into the released alpine images at + # runtime (curl/ssh/dig/python3/getent). A nixosTest has no network, so we bake + # them in at build time instead — one buildEnv copied onto the image so they + # land on /bin (already on the alpine PATH), no per-tool symlinks. Only /bin is + # linked: overlaying /lib would shadow musl's loader and break alpine's + # tailscaled. The binaries carry their own glibc closure via RPATH, so they run + # on musl without it. Tests assert these are present rather than installing them. + testTools = pkgs.buildEnv { + name = "ts-test-tools"; + paths = with pkgs; [ curl openssh dnsutils python3 getent ]; + pathsToLink = [ "/bin" ]; + }; + + # Pinned registry images, as docker-loadable tarballs that keep their + # repo:tag so the suite finds them locally (its own pull fails offline, then + # RunWithOptions uses the loaded image). The spec's sha256 is per-arch. + pullImage = spec: pkgs.dockerTools.pullImage { + imageName = spec.imageName; + finalImageName = spec.imageName; + finalImageTag = spec.finalImageTag; + imageDigest = spec.imageDigest; + os = "linux"; + arch = dockerArch; + sha256 = spec.sha256.${dockerArch}; + }; + + # Released tailscale images with the test tools layered on, keeping the exact + # registry repo:tag the suite requests, and the SSH login pre-created at build + # time (alpine's busybox adduser) so no container is mutated at startup — + # matching the head image's baked ssh-it-user. + withTestTools = spec: pkgs.dockerTools.buildImage { + name = spec.imageName; + tag = spec.finalImageTag; + fromImage = pullImage spec; + copyToRoot = testTools; + runAsRoot = '' + #!${pkgs.runtimeShell} + export PATH=/usr/sbin:/usr/bin:/sbin:/bin + ${lib.concatMapStringsSep "\n" (u: "grep -q '^${u}:' /etc/passwd || adduser -D -s /bin/sh ${u}") sshTestUsers} + ''; + }; + + tailscaleVersionImages = map withTestTools (builtins.attrValues versions.images); + postgresImage = pullImage versions.postgres; +in +{ + inherit + tailscaleImage + headscaleImage + postgresImage + tailscaleVersionImages + tailscaleHead + caShim; +} From bdb9cc510cd61ff1d1523660f859c0c8d011a7cd Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 25 Jun 2026 08:06:37 +0000 Subject: [PATCH 3/7] integration: support prebuilt images and the slow nix VM in the suite Read the HEADSCALE_INTEGRATION_*_IMAGE knobs through one envknob helper and assert required tooling is present instead of installing it at runtime (the nix VM is offline). dsic waits for both the derper cert and key before starting derper, so the container can't die between the two uploads. The nested-docker VM is slower than a host daemon, so every integration timeout (convergence, docker exec, peer sync, pool wait) now flows through one dockertestutil.ScaleTimeout honouring HEADSCALE_INTEGRATION_TIMEOUT_SCALE. --- integration/dockertestutil/execute.go | 32 ++++++-- integration/dsic/dsic.go | 79 +++++++++++++------- integration/general_test.go | 11 +-- integration/helpers.go | 9 +-- integration/hsic/hsic.go | 33 ++++----- integration/integrationutil/util.go | 59 +++++++++++---- integration/scenario.go | 103 +++++++++++++++----------- integration/ssh_test.go | 8 +- integration/tsic/tsic.go | 75 +++++++++++++------ integration/tsric/tsric.go | 10 +-- 10 files changed, 258 insertions(+), 161 deletions(-) diff --git a/integration/dockertestutil/execute.go b/integration/dockertestutil/execute.go index 79511e2e0..d49ba6455 100644 --- a/integration/dockertestutil/execute.go +++ b/integration/dockertestutil/execute.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "fmt" + "os" + "strconv" "sync" "time" @@ -11,15 +13,31 @@ import ( "github.com/ory/dockertest/v3" ) -// defaultExecuteTimeout returns the timeout for docker exec commands. -// On CI runners, docker exec latency is higher due to resource -// contention, so the timeout is doubled. -func defaultExecuteTimeout() time.Duration { - if util.IsCI() { - return 20 * time.Second +// ScaleTimeout scales an integration timeout for the running environment: the +// nested-docker nix-VM checks are slower than a host docker daemon and set +// HEADSCALE_INTEGRATION_TIMEOUT_SCALE to widen every budget; otherwise CI uses +// 2x and local dev the base. One knob covers exec, convergence, peer sync, and +// pool waits. Lives here (the lowest integration package) so every layer can +// reach it. +func ScaleTimeout(base time.Duration) time.Duration { + scale := os.Getenv("HEADSCALE_INTEGRATION_TIMEOUT_SCALE") + if scale != "" { + f, err := strconv.ParseFloat(scale, 64) + if err == nil && f > 0 { + return time.Duration(float64(base) * f) + } } - return 10 * time.Second + if util.IsCI() { + return base * 2 + } + + return base +} + +// defaultExecuteTimeout returns the timeout for docker exec commands. +func defaultExecuteTimeout() time.Duration { + return ScaleTimeout(10 * time.Second) } var ( diff --git a/integration/dsic/dsic.go b/integration/dsic/dsic.go index c165eee10..3d531fa7e 100644 --- a/integration/dsic/dsic.go +++ b/integration/dsic/dsic.go @@ -117,9 +117,14 @@ func (dsic *DERPServerInContainer) buildEntrypoint(derperArgs string) []string { // Wait for network to be ready commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done") - // Wait for TLS cert to be written (always written after container start) + // Wait for the TLS cert AND key to be written (both written after container + // start). Waiting only for the cert races: once it lands derper starts, and + // since it also needs the key — written a moment later — it exits and kills + // the container before the key upload (reliably, with slow nested-docker exec). commands = append(commands, fmt.Sprintf("while [ ! -f %s/%s.crt ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname)) + commands = append(commands, + fmt.Sprintf("while [ ! -f %s/%s.key ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname)) // If CA certs are configured, wait for them to be written if len(dsic.caCerts) > 0 { @@ -221,34 +226,56 @@ func New( var container *dockertest.Resource - buildOptions := &dockertest.BuildOptions{ - Dockerfile: "Dockerfile.derper", - ContextDir: dockerContextPath, - BuildArgs: []docker.BuildArg{}, - } - - switch version { - case "head": - buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ - Name: "VERSION_BRANCH", - Value: "main", - }) - default: - buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ - Name: "VERSION_BRANCH", - Value: "v" + version, - }) - } // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(runOptions, "derp") - container, err = pool.BuildAndRunWithBuildOptions( - buildOptions, - runOptions, - dockertestutil.DockerRestartPolicy, - dockertestutil.DockerAllowLocalIPv6, - dockertestutil.DockerAllowNetworkAdministration, - ) + // In CI / nix checks a prebuilt derper image is provided so we neither build + // the image nor clone tailscale at runtime, mirroring the tailscale and + // headscale image injection. Only the head image is prebuilt. + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_DERPER_IMAGE") + if err != nil { + return nil, err + } + + if prebuilt && version == "head" { + runOptions.Repository = repo + runOptions.Tag = tag + + container, err = pool.RunWithOptions( + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + } else { + buildOptions := &dockertest.BuildOptions{ + Dockerfile: "Dockerfile.derper", + ContextDir: dockerContextPath, + BuildArgs: []docker.BuildArg{}, + } + + switch version { + case "head": + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "main", + }) + default: + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "v" + version, + }) + } + + container, err = pool.BuildAndRunWithBuildOptions( + buildOptions, + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + } + if err != nil { return nil, fmt.Errorf( "%s starting tailscale DERPer container (version: %s): %w", diff --git a/integration/general_test.go b/integration/general_test.go index 9bdfd8c71..689c9a308 100644 --- a/integration/general_test.go +++ b/integration/general_test.go @@ -483,16 +483,7 @@ func TestTaildrop(t *testing.T) { _, err = scenario.ListTailscaleClientsFQDNs() requireNoErrListFQDN(t, err) - // Install curl on all clients - for _, client := range allClients { - if !strings.Contains(client.Hostname(), "head") { - command := []string{"apk", "add", "curl"} - _, _, err := client.Execute(command) - if err != nil { - t.Fatalf("failed to install curl on %s, err: %s", client.Hostname(), err) - } - } - } + // curl is baked into every image (no runtime install). // Helper to get FileTargets for a client. getFileTargets := func(client TailscaleClient) ([]apitype.FileTarget, error) { diff --git a/integration/helpers.go b/integration/helpers.go index 908c7447b..859d805fd 100644 --- a/integration/helpers.go +++ b/integration/helpers.go @@ -22,6 +22,7 @@ import ( policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/hscontrol/util" + "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/integrationutil" "github.com/juanfont/headscale/integration/tsic" "github.com/oauth2-proxy/mockoidc" @@ -922,13 +923,7 @@ func assertCommandOutputContains(t *testing.T, c TailscaleClient, command []stri // Uses longer timeouts in CI environments to account for slower resource allocation // and higher system load during automated testing. func dockertestMaxWait() time.Duration { - wait := 300 * time.Second //nolint - - if util.IsCI() { - wait = 600 * time.Second //nolint - } - - return wait + return dockertestutil.ScaleTimeout(300 * time.Second) } // didClientUseWebsocketForDERP analyzes client logs to determine if WebSocket was used for DERP. diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index d37bac9af..58de979c0 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -57,10 +57,8 @@ const ( ) var ( - errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") - errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag") - errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI") - errInvalidPostgresImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_POSTGRES_IMAGE format, expected repository:tag") + errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") + errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI") ) type fileInContainer struct { @@ -406,12 +404,12 @@ func New( pgRepo := "postgres" pgTag := "latest" - if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_POSTGRES_IMAGE"); prebuiltImage != "" { - repo, tag, found := strings.Cut(prebuiltImage, ":") - if !found { - return nil, errInvalidPostgresImageFormat - } + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_POSTGRES_IMAGE") + if err != nil { + return nil, err + } + if prebuilt { pgRepo = repo pgTag = tag } @@ -510,16 +508,13 @@ func New( var container *dockertest.Resource // Check if a pre-built image is available via environment variable - prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") - - if prebuiltImage != "" { - log.Printf("Using pre-built headscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value - // Parse image into repository and tag - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, errInvalidHeadscaleImageFormat - } + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") + if err != nil { + return nil, err + } + if prebuilt { + log.Printf("Using pre-built headscale image: %s:%s", repo, tag) //nolint:gosec // G706: integration-only log of trusted env value runOptions.Repository = repo runOptions.Tag = tag @@ -530,7 +525,7 @@ func New( dockertestutil.DockerAllowNetworkAdministration, ) if err != nil { - return nil, fmt.Errorf("running pre-built headscale container %q: %w", prebuiltImage, err) + return nil, fmt.Errorf("running pre-built headscale container %s:%s: %w", repo, tag, err) } } else if util.IsCI() { return nil, errHeadscaleImageRequiredInCI diff --git a/integration/integrationutil/util.go b/integration/integrationutil/util.go index a70145e0c..f31357c75 100644 --- a/integration/integrationutil/util.go +++ b/integration/integrationutil/util.go @@ -8,28 +8,61 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "fmt" "io" "math/big" "path/filepath" + "strings" "time" "github.com/juanfont/headscale/hscontrol/types" - "github.com/juanfont/headscale/hscontrol/util" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/tailcfg" ) -// PeerSyncTimeout returns the timeout for peer synchronization based on environment: -// 60s for dev, 120s for CI. -func PeerSyncTimeout() time.Duration { - if util.IsCI() { - return 120 * time.Second +var errInvalidImageFormat = errors.New("integration image env must be in repository:tag format") + +// PrebuiltImage reads a HEADSCALE_INTEGRATION_*_IMAGE knob (the full var name, +// e.g. "HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") and splits it into repository +// and tag. The bool is false when the knob is unset — the suite then builds the +// image itself; it errors when the value is set but is not "repository:tag". +// This is the one place the prebuilt-image knobs (used by the CI / nix-check +// path) are read, via tailscale's envknob. +func PrebuiltImage(envVar string) (string, string, bool, error) { + image := envknob.String(envVar) + if image == "" { + return "", "", false, nil } - return 60 * time.Second + repo, tag, err := ParseImageRef(image) + if err != nil { + return "", "", false, fmt.Errorf("%s=%w", envVar, err) + } + + return repo, tag, true, nil +} + +// ParseImageRef splits an already-resolved "repository:tag" image string into +// its parts. Use it where the image comes from somewhere other than a single +// knob (e.g. a websocket-tagged override chosen at runtime); for the common +// read-a-knob case use [PrebuiltImage]. +func ParseImageRef(image string) (string, string, error) { + repo, tag, found := strings.Cut(image, ":") + if !found { + return "", "", fmt.Errorf("%q: %w", image, errInvalidImageFormat) + } + + return repo, tag, nil +} + +// PeerSyncTimeout returns the timeout for peer synchronization: 60s for dev, +// scaled for CI / the slow nix VM. +func PeerSyncTimeout() time.Duration { + return dockertestutil.ScaleTimeout(60 * time.Second) } // PeerSyncRetryInterval returns the retry interval for peer synchronization checks. @@ -37,16 +70,10 @@ func PeerSyncRetryInterval() time.Duration { return 100 * time.Millisecond } -// ScaledTimeout returns the given timeout, scaled for CI environments -// where resource contention causes slower state propagation. -// Uses a 2x multiplier, consistent with PeerSyncTimeout (60s/120s) -// and dockertestMaxWait (300s/600s). +// ScaledTimeout returns the given convergence budget, scaled for the running +// environment via [dockertestutil.ScaleTimeout]. func ScaledTimeout(d time.Duration) time.Duration { - if util.IsCI() { - return d * 2 - } - - return d + return dockertestutil.ScaleTimeout(d) } func WriteFileToContainer( diff --git a/integration/scenario.go b/integration/scenario.go index 7af281619..58cbeb72d 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -3,6 +3,7 @@ package integration import ( "context" "crypto/tls" + _ "embed" "encoding/json" "errors" "fmt" @@ -22,7 +23,6 @@ import ( "time" clientv1 "github.com/juanfont/headscale/gen/client/v1" - "github.com/juanfont/headscale/hscontrol/capver" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/dsic" @@ -54,30 +54,38 @@ var ( errNoHeadscaleAvailable = errors.New("no headscale available") errNoUserAvailable = errors.New("no user available") errNoClientFound = errors.New("client not found") - - // AllVersions represents a list of Tailscale versions the suite - // uses to test compatibility with the [ControlServer]. - // - // The list contains two special cases, "head" and "unstable" which - // points to the current tip of Tailscale's main branch and the latest - // released unstable version. - // - // The rest of the version represents Tailscale versions that can be - // found in Tailscale's apt repository. - AllVersions = append([]string{"head", "unstable"}, capver.TailscaleLatestMajorMinor(capver.SupportedMajorMinorVersions, true)...) - - // MustTestVersions is the minimum set of versions we should test. - // At the moment, this is arbitrarily chosen as: - // - // - Two unstable (HEAD and unstable) - // - Two latest versions - // - Two oldest supported version. - MustTestVersions = append( - AllVersions[0:4], - AllVersions[len(AllVersions)-2:]..., - ) ) +// Regenerate tailscale-versions.json from capver + nix-prefetch-docker. Runs in +// the same `make generate` / `go generate ./...` pass as capver (and needs the +// nix dev shell); already-pinned tags are reused, so an unchanged set is a +// no-op. Refresh just the pins with `go generate ./integration/...`. +//go:generate go run ../tools/tailscale-versions + +//go:embed tailscale-versions.json +var tailscaleVersionsJSON []byte + +// MustTestVersions is the set of Tailscale versions the suite tests for +// compatibility with the [ControlServer]. It contains "head" (built from +// source) and "unstable", plus the latest and oldest supported releases. It is +// read from the generated tailscale-versions.json — the single source shared +// with the nix image pins (regenerate with `update-integration-images`, which +// derives the list from capver). +var MustTestVersions = mustTailscaleVersions() + +func mustTailscaleVersions() []string { + var v struct { + Versions []string `json:"versions"` + } + + err := json.Unmarshal(tailscaleVersionsJSON, &v) + if err != nil { + panic("parsing tailscale-versions.json: " + err.Error()) + } + + return v.Versions +} + // User represents a User in the [ControlServer] and a map of [TailscaleClient]'s // associated with the User. type User struct { @@ -1583,6 +1591,33 @@ const ( var errStatusCodeNotOK = errors.New("status code not OK") +// runHeadscaleImageContainer starts runOptions from the prebuilt headscale +// image when HEADSCALE_INTEGRATION_HEADSCALE_IMAGE is set, otherwise it builds +// Dockerfile.integration. The mock OIDC provider (headscale mockoidc) and the +// webservice (python3 -m http.server) both run on the headscale image, so they +// share the same prebuilt-or-build path hsic uses for headscale itself — which +// is what lets them come up offline (e.g. in the nix VM checks). +func (s *Scenario) runHeadscaleImageContainer(runOptions *dockertest.RunOptions) (*dockertest.Resource, error) { + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") + if err != nil { + return nil, err + } + + if prebuilt { + runOptions.Repository = repo + runOptions.Tag = tag + + return s.pool.RunWithOptions(runOptions, dockertestutil.DockerRestartPolicy) + } + + buildOptions := &dockertest.BuildOptions{ + Dockerfile: hsic.IntegrationTestDockerFileName, + ContextDir: dockerContextPath, + } + + return s.pool.BuildAndRunWithBuildOptions(buildOptions, runOptions, dockertestutil.DockerRestartPolicy) +} + func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUser) error { port, err := dockertestutil.RandomFreeHostPort() if err != nil { @@ -1618,11 +1653,6 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse }, } - headscaleBuildOptions := &dockertest.BuildOptions{ - Dockerfile: hsic.IntegrationTestDockerFileName, - ContextDir: dockerContextPath, - } - err = s.pool.RemoveContainerByName(hostname) if err != nil { return err @@ -1633,11 +1663,7 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(mockOidcOptions, "oidc") - if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( //nolint:noinlineerr - headscaleBuildOptions, - mockOidcOptions, - dockertestutil.DockerRestartPolicy, - ); err == nil { + if pmockoidc, err := s.runHeadscaleImageContainer(mockOidcOptions); err == nil { //nolint:noinlineerr s.mockOIDC.r = pmockoidc } else { return err @@ -1722,16 +1748,7 @@ func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) { // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(webOpts, "web") - webBOpts := &dockertest.BuildOptions{ - Dockerfile: hsic.IntegrationTestDockerFileName, - ContextDir: dockerContextPath, - } - - web, err := s.pool.BuildAndRunWithBuildOptions( - webBOpts, - webOpts, - dockertestutil.DockerRestartPolicy, - ) + web, err := s.runHeadscaleImageContainer(webOpts) if err != nil { return nil, err } diff --git a/integration/ssh_test.go b/integration/ssh_test.go index 7ed50c206..fb14f8cd4 100644 --- a/integration/ssh_test.go +++ b/integration/ssh_test.go @@ -47,7 +47,6 @@ func sshScenario(t *testing.T, policy *policyv2.Policy, testName string, clients // to not configure DNS. tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(policy), @@ -445,7 +444,7 @@ func doSSHWithRetryAsUser( peerFQDN, _ := peer.FQDN() command := []string{ - "/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1", + "ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1", fmt.Sprintf("%s@%s", sshUser, peerFQDN), "'hostname'", } @@ -661,7 +660,7 @@ func doSSHCheckWithTimeout( peerFQDN, _ := peer.FQDN() command := []string{ - "/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30", + "ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30", fmt.Sprintf("%s@%s", "ssh-it-user", peerFQDN), "'hostname'", } @@ -935,7 +934,6 @@ func TestSSHOneUserToOneCheckModeOIDC(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(sshCheckPolicy()), @@ -1035,7 +1033,6 @@ func TestSSHCheckModeUnapprovedTimeout(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(sshCheckPolicy()), @@ -1607,7 +1604,6 @@ func TestSSHLocalpart(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser user1", "adduser user2"), tsic.WithDockerWorkdir("/"), }, hsic.WithTestName("sshlocalpart"), diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 3e19d2b14..6770c837e 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -26,6 +26,7 @@ import ( "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store/mem" @@ -65,7 +66,6 @@ var ( errTailscaleWrongPeerCount = errors.New("wrong peer count") errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey") errInvalidClientConfig = errors.New("verifiably invalid client config requested") - errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag") errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version") errContainerNotInitialized = errors.New("container not initialized") errFQDNNotYetAvailable = errors.New("FQDN not yet available") @@ -244,20 +244,33 @@ func WithAcceptRoutes() Option { } } -// WithPackages specifies Alpine packages to install when the container starts. -// This requires internet access and uses `apk add`. Common packages: +// WithPackages declares the tools a test needs inside the container. The images +// bake these in (no runtime install), so this asserts they are present at +// startup and fails the container loudly if not. Common packages: // - "python3" for HTTP server // - "curl" for HTTP client // - "bind-tools" for dig command -// - "iptables", "ip6tables" for firewall rules -// Note: Tests using this option require internet access and cannot use -// the built-in DERP server in offline mode. +// - "openssh" for the ssh client. func WithPackages(packages ...string) Option { return func(tsic *TailscaleInContainer) { tsic.withPackages = append(tsic.withPackages, packages...) } } +// packageBinary maps an alpine package name to a representative binary the +// images must provide, so [WithPackages] can assert the binary is on PATH +// rather than installing the package at runtime. +func packageBinary(pkg string) string { + switch pkg { + case "openssh": + return "ssh" + case "bind-tools": + return "dig" + default: + return pkg + } +} + // WithWebserver starts a Python HTTP server on the specified port // alongside tailscaled. This is useful for testing subnet routing // and ACL connectivity. Automatically adds "python3" to packages if needed. @@ -285,20 +298,31 @@ func (t *TailscaleInContainer) buildEntrypoint() []string { commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done") // If CA certs are configured, wait for them to be written by the Go code - // (certs are written after container start via [TailscaleInContainer.WriteFile]) - if len(t.caCerts) > 0 { + // (certs are written after container start via [TailscaleInContainer.WriteFile]). + // Wait for the LAST cert (highest index): they are written in order, so once + // user-{N-1}.crt exists every earlier one does too. Waiting only for user-0 + // races update-ca-certificates ahead of later certs (e.g. the headscale CA at + // user-1 when a DERP cert occupies user-0), permanently skipping their trust. + if n := len(t.caCerts); n > 0 { commands = append(commands, - fmt.Sprintf("while [ ! -f %s/user-0.crt ]; do sleep 0.1; done", caCertRoot)) + fmt.Sprintf("while [ ! -f %s/user-%d.crt ]; do sleep 0.1; done", caCertRoot, n-1)) } - // Install packages if requested (requires internet access) + // The images bake in every tool the suite needs (nix/images.nix for the head + // and version images, the alpine base otherwise), so assert the requested + // packages are present rather than installing them — a hermetic/offline run + // has no package mirror. Fail the container loudly if one is missing so an + // image regression is obvious in its log. packages := t.withPackages if t.withWebserverPort > 0 && !slices.Contains(packages, "python3") { packages = append(packages, "python3") } - if len(packages) > 0 { - commands = append(commands, "apk add --no-cache "+strings.Join(packages, " ")) + for _, pkg := range packages { + bin := packageBinary(pkg) + commands = append(commands, fmt.Sprintf( + "command -v %s >/dev/null 2>&1 || { echo 'tsic: required tool %s (package %s) missing from image' >&2; exit 1; }", + bin, bin, pkg)) } // Update CA certificates @@ -427,24 +451,31 @@ func New( switch version { case VersionHead: // Check if a pre-built image is available via environment variable - prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE") + prebuiltImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE") - // If custom build tags are required (e.g., for websocket DERP), we cannot use - // the pre-built image as it won't have the necessary code compiled in. + // If custom build tags are required (e.g., for websocket DERP), the + // default pre-built image won't have the necessary code compiled in. A + // websocket-tagged prebuilt image can be injected separately (CI / nix + // checks build one with ts_debug_websockets); otherwise fall back to + // building the image with the tags. hasBuildTags := len(tsic.buildConfig.tags) > 0 if hasBuildTags && prebuiltImage != "" { - log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value - prebuiltImage, tsic.buildConfig.tags) - prebuiltImage = "" + wsImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE") + if tsic.withWebsocketDERP && wsImage != "" { + prebuiltImage = wsImage + } else { + log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value + prebuiltImage, tsic.buildConfig.tags) + prebuiltImage = "" + } } if prebuiltImage != "" { log.Printf("Using pre-built tailscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value - // Parse image into repository and tag - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, errInvalidTailscaleImageFormat + repo, tag, err := integrationutil.ParseImageRef(prebuiltImage) + if err != nil { + return nil, err } tailscaleOptions.Repository = repo diff --git a/integration/tsric/tsric.go b/integration/tsric/tsric.go index e160d855d..fdeef4942 100644 --- a/integration/tsric/tsric.go +++ b/integration/tsric/tsric.go @@ -12,13 +12,13 @@ import ( "fmt" "io" "log" - "os" "strings" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/util/rands" ) @@ -35,7 +35,7 @@ const ( // getPrebuiltImage returns the pre-built tailscale-rs Docker image name if set. func getPrebuiltImage() string { - return os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE") + return envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE") } // TailscaleRustInContainer runs the tailscale-rs axum example as an @@ -210,9 +210,9 @@ func New( if prebuiltImage := getPrebuiltImage(); prebuiltImage != "" { log.Printf("Using pre-built tailscale-rs image: %s", prebuiltImage) - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, fmt.Errorf("tsric: invalid image format %q, expected repository:tag", prebuiltImage) //nolint:err113 + repo, tag, err := integrationutil.ParseImageRef(prebuiltImage) + if err != nil { + return nil, fmt.Errorf("tsric: %w", err) } runOptions.Repository = repo From 70b0c9208b1fbb72bca231d243fc7632eaa27af2 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 25 Jun 2026 08:07:22 +0000 Subject: [PATCH 4/7] nix: run each integration test as a NixOS-VM flake check Each test becomes a hermetic check: a VM boots docker, loads the prebuilt images and runs that one test (nix/tests/integration.nix), so the suite runs offline as nix build .#checks.. The generator emits the test list as nix: heavy multi-subtest tests are split into per-subtest checks, and tests the nested docker VM can't host are excluded for the legacy pipeline. flake.nix wires the checks, the images, and the Go build/test/lint checks (with the version JSON on the embed whitelist). --- .../gh-action-integration-generator.go | 167 +++++++++++------- flake.lock | 70 +++++++- flake.nix | 138 ++++++++++++++- integration/excluded-tests.json | 7 + integration/postgres-tests.nix | 8 + integration/tests.nix | 156 ++++++++++++++++ nix/tests/integration.nix | 157 ++++++++++++++++ 7 files changed, 637 insertions(+), 66 deletions(-) create mode 100644 integration/excluded-tests.json create mode 100644 integration/postgres-tests.nix create mode 100644 integration/tests.nix create mode 100644 nix/tests/integration.nix diff --git a/.github/workflows/gh-action-integration-generator.go b/.github/workflows/gh-action-integration-generator.go index 5bb8318bd..cc9c069c0 100644 --- a/.github/workflows/gh-action-integration-generator.go +++ b/.github/workflows/gh-action-integration-generator.go @@ -4,30 +4,49 @@ package main import ( "bytes" + "encoding/json" "fmt" "log" + "os" "os/exec" + "slices" "strings" ) -// testsToSplit defines tests that should be split into multiple CI jobs. -// Key is the test function name, value is a list of subtest prefixes. -// Each prefix becomes a separate CI job as "TestName/prefix". -// -// Example: [TestAutoApproveMultiNetwork] has subtests like: -// - TestAutoApproveMultiNetwork/authkey-tag-advertiseduringup-false-pol-database -// - TestAutoApproveMultiNetwork/webauth-user-advertiseduringup-true-pol-file -// -// Splitting by approver type (tag, user, group) creates 6 CI jobs with 4 tests each: -// - TestAutoApproveMultiNetwork/authkey-tag.* (4 tests) -// - TestAutoApproveMultiNetwork/authkey-user.* (4 tests) -// - TestAutoApproveMultiNetwork/authkey-group.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-tag.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-user.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-group.* (4 tests) -// -// This reduces load per CI job (4 tests instead of 12) to avoid infrastructure -// flakiness when running many sequential Docker-based integration tests. +// excludedFromNixChecks lists tests the hermetic nix-VM checks cannot run, with +// the reason. They still run via cmd/hi, where the missing resource exists. +var excludedFromNixChecks = map[string]string{ + // Verifies connectivity through Tailscale's *public* DERP relays + // (HEADSCALE_DERP_URLS=controlplane.tailscale.com). A hermetic offline VM + // has no internet and the nix sandbox forbids network. + "TestPingAllByIPPublicDERP": "requires public internet DERP", + + // Also routes DERP through Tailscale's public relays: tailscale-rs cannot + // validate the embedded DERP's self-signed cert (WithPublicDERP in the + // test), so it needs the same internet access PublicDERP does. + "TestTailscaleRustAxum": "requires public internet DERP (tailscale-rs)", + + // These drive `docker network disconnect`/`connect` to simulate cable pulls. + // In the VM's nested docker, libnetwork intermittently fails the reconnect + // ("failed to set gateway: network is unreachable" — a stale IPAM/endpoint + // reservation), which does not happen against a host docker daemon. They run + // reliably via cmd/hi in the legacy pipeline. + "TestHASubnetRouterFailoverDockerDisconnect": "docker reconnect unreliable in nested VM docker", + "TestHASubnetRouterFailoverBothOfflineCablePull": "docker reconnect unreliable in nested VM docker", + + // Ping-driven HA failover: after a router recovers it asserts router 2 stays + // primary and traffic still traceroutes through it. In the VM's nested docker + // that data-plane state never converges (a 4x ScaledTimeout budget did not + // help — behaviour, not timing); it holds against a host docker daemon. + "TestHASubnetRouterPingFailover": "ping-failover data plane doesn't converge in nested VM docker", +} + +// testsToSplit defines tests whose subtests are run as separate checks. Key is +// the test function, value is a list of subtest prefixes; each prefix becomes +// its own check as "TestName/prefix.*". TestAutoApproveMultiNetwork fans out +// over approver × policy-mode × advertise (24 subtests); running all of them in +// one nested-docker VM check overruns the timeout, so split by approver into +// 6 checks of 4 — fewer sequential docker scenarios per check. var testsToSplit = map[string][]string{ "TestAutoApproveMultiNetwork": { "authkey-tag", @@ -39,26 +58,32 @@ var testsToSplit = map[string][]string{ }, } -// expandTests takes a list of test names and expands any that need splitting -// into multiple subtest patterns. +// expandTests replaces any splittable test with one entry per subtest prefix. +// The ".*" lets the check's "^...$"-wrapped -test.run match the full subtest +// names (e.g. authkey-tag-advertiseduringup-false-pol-database). func expandTests(tests []string) []string { - var expanded []string + expanded := make([]string, 0, len(tests)) for _, test := range tests { - if prefixes, ok := testsToSplit[test]; ok { - // This test should be split into multiple jobs. - // We append ".*" to each prefix because the CI runner wraps patterns - // with ^...$ anchors. Without ".*", a pattern like "authkey$" wouldn't - // match "authkey-tag-advertiseduringup-false-pol-database". - for _, prefix := range prefixes { - expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix)) - } - } else { + prefixes, ok := testsToSplit[test] + if !ok { expanded = append(expanded, test) + continue + } + + for _, prefix := range prefixes { + expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix)) } } + return expanded } +// This generator enumerates the integration test functions and writes them as +// nix lists (integration/tests.nix and integration/postgres-tests.nix). The +// flake maps each entry to a NixOS-VM check (sqlite for all, plus a postgres +// variant for the subset) — see nix/tests/integration.nix. One check per test +// function; the GitHub workflow derives its matrix from these checks. + func findTests() []string { rgBin, err := exec.LookPath("rg") if err != nil { @@ -89,43 +114,41 @@ func findTests() []string { return tests } -func updateYAML(tests []string, jobName string, testPath string) { - testsForYq := fmt.Sprintf("[%s]", strings.Join(tests, ", ")) +// writeNixList writes the test function names as a nix list, consumed by +// flake.nix to generate one integration check per test. +func writeNixList(tests []string, nixPath string) { + var b strings.Builder + b.WriteString("# Generated by gh-action-integration-generator.go; do not edit.\n") + b.WriteString("[\n") + for _, test := range tests { + fmt.Fprintf(&b, " %q\n", test) + } + b.WriteString("]\n") - yqCommand := fmt.Sprintf( - "yq eval '.jobs.%s.strategy.matrix.test = %s' %s -i", - jobName, - testsForYq, - testPath, - ) - cmd := exec.Command("bash", "-c", yqCommand) - - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err := cmd.Run() - if err != nil { - log.Printf("stdout: %s", stdout.String()) - log.Printf("stderr: %s", stderr.String()) - log.Fatalf("failed to run yq command: %s", err) + if err := os.WriteFile(nixPath, []byte(b.String()), 0o644); err != nil { + log.Fatalf("failed to write %s: %s", nixPath, err) } - fmt.Printf("YAML file (%s) job %s updated successfully\n", testPath, jobName) + fmt.Printf("Nix list (%s) updated successfully\n", nixPath) } func main() { tests := findTests() - // Expand tests that should be split into multiple jobs - expandedTests := expandTests(tests) + tests = slices.DeleteFunc(tests, func(name string) bool { + if reason, ok := excludedFromNixChecks[name]; ok { + log.Printf("excluding %s from nix checks: %s", name, reason) + return true + } + return false + }) - quotedTests := make([]string, len(expandedTests)) - for i, test := range expandedTests { - quotedTests[i] = fmt.Sprintf("\"%s\"", test) - } + // Split heavy tests into per-subtest checks (after exclusion, so an excluded + // test is never expanded). + tests = expandTests(tests) - // Define selected tests for PostgreSQL + // Selected tests also run against PostgreSQL to verify database + // compatibility (a postgres-variant check per name). postgresTestNames := []string{ "TestACLAllowUserDst", "TestPingAllByIP", @@ -134,12 +157,30 @@ func main() { "TestSubnetRouterMultiNetwork", } - quotedPostgresTests := make([]string, len(postgresTestNames)) - for i, test := range postgresTestNames { - quotedPostgresTests[i] = fmt.Sprintf("\"%s\"", test) + // Tests excluded from the hermetic nix checks still need to run somewhere, + // so emit them for the legacy cmd/hi GitHub-Actions matrix (real internet). + excluded := make([]string, 0, len(excludedFromNixChecks)) + for name := range excludedFromNixChecks { + excluded = append(excluded, name) + } + slices.Sort(excluded) + + writeNixList(tests, "../../integration/tests.nix") + writeNixList(postgresTestNames, "../../integration/postgres-tests.nix") + writeJSONList(excluded, "../../integration/excluded-tests.json") +} + +// writeJSONList writes the names as a JSON array, consumed as a GitHub-Actions +// matrix by the legacy cmd/hi workflow. +func writeJSONList(names []string, path string) { + data, err := json.MarshalIndent(names, "", " ") + if err != nil { + log.Fatalf("marshalling %s: %s", path, err) } - // Update both SQLite and PostgreSQL job matrices - updateYAML(quotedTests, "sqlite", "./test-integration.yaml") - updateYAML(quotedPostgresTests, "postgres", "./test-integration.yaml") + if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil { + log.Fatalf("failed to write %s: %s", path, err) + } + + fmt.Printf("JSON list (%s) updated successfully\n", path) } diff --git a/flake.lock b/flake.lock index c723d287a..86f9ce4c7 100644 --- a/flake.lock +++ b/flake.lock @@ -22,6 +22,22 @@ "type": "github" } }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -74,11 +90,28 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1772736753, + "narHash": "sha256-au/m3+EuBLoSzWUCb64a/MZq6QUtOV8oC0D9tY2scPQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "917fec990948658ef1ccd07cef2a1ef060786846", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake-checks": "flake-checks", "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "tailscale-head": "tailscale-head" } }, "systems": { @@ -111,6 +144,41 @@ "type": "github" } }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "tailscale-head": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs_2", + "systems": "systems_3" + }, + "locked": { + "lastModified": 1782250635, + "narHash": "sha256-y4/EAApBPekjWz2cP/9RexTBkiSAXsdOHWnhm/no/4I=", + "owner": "tailscale", + "repo": "tailscale", + "rev": "d4f2917c1bfd27529ec7997d08a8f1aa9ea903f2", + "type": "github" + }, + "original": { + "owner": "tailscale", + "repo": "tailscale", + "type": "github" + } + }, "treefmt-nix": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index 96786def3..91cc9f8c8 100644 --- a/flake.nix +++ b/flake.nix @@ -9,10 +9,18 @@ # once it ships go_1_26 >= 1.26.4. nixpkgs.url = "github:NixOS/nixpkgs/staging-next-26.05"; flake-utils.url = "github:numtide/flake-utils"; + # Reusable Go flake checks (build/test/lint/format); CI runs them via # `nix build .#checks..` instead of bespoke per-tool steps. flake-checks.url = "github:kradalby/flake-checks"; flake-checks.inputs.nixpkgs.follows = "nixpkgs"; + + # Tailscale HEAD, built from source for the integration test client image. + # Bump with `nix flake update tailscale-head` to track the latest main. + # It is a flake; we still build our own derivation (we need derper + + # containerboot, which its default package omits) but read its committed + # vendorHash from flakehashes.json so the bump carries the hash. + tailscale-head.url = "github:tailscale/tailscale"; }; outputs = @@ -20,6 +28,7 @@ , nixpkgs , flake-utils , flake-checks + , tailscale-head , ... }: let @@ -39,8 +48,27 @@ # Go 1.26 builder; resolves to Go 1.26.4 from the pinned nixpkgs. buildGo = pkgs.buildGo126Module; vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri; + + # Go source with the non-Go scaffolding filtered out, so editing the + # nix integration harness, CI workflows, docs or packaging does not + # rebuild the (slow) integration test binary or container images. + # Only changes to actual Go source invalidate them. + goSrc = pkgs.lib.fileset.toSource { + root = ./.; + fileset = pkgs.lib.fileset.difference ./. (pkgs.lib.fileset.unions [ + ./nix + ./.github + ./docs + ./packaging + ./flake.nix + ./flake.lock + ./flakehashes.json + ]); + }; in { + headscale-go-src = goSrc; + headscale = buildGo { pname = "headscale"; version = headscaleVersion; @@ -71,6 +99,34 @@ subPackages = [ "cmd/hi" ]; }; + # The integration suite compiled into a single test binary, run later + # inside a per-test NixOS VM check (see nix/tests/integration.nix). + # Compiled once and shared by every integration check. + integration-test-bin = buildGo { + pname = "headscale-integration-test"; + # Fixed (not the commit rev) so the binary is content-addressed: + # identical Go source → identical store path → shared-cache hit + # across commits. The version string is irrelevant for a test binary. + version = "integration"; + src = goSrc; + + inherit vendorHash; + + doCheck = false; + + buildPhase = '' + runHook preBuild + go test -c -o integration.test ./integration + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + install -Dm755 integration.test $out/bin/integration.test + runHook postInstall + ''; + }; + # Build golangci-lint with stock Go 1.26 (upstream uses hardcoded Go # version); it does not build against the pinned 1.26.4. golangci-lint = buildGo rec { @@ -200,7 +256,12 @@ vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri; goPkg = pkgs.go_1_26; # //go:embed targets and test-read files outside the default whitelist. - embedDirs = [ ./hscontrol/assets ./hscontrol/db/schema.sql ./config-example.yaml ]; + embedDirs = [ + ./hscontrol/assets + ./hscontrol/db/schema.sql + ./config-example.yaml + ./integration/tailscale-versions.json + ]; extraSrc = [ ./hscontrol/testdata ./hscontrol/types/testdata @@ -239,6 +300,45 @@ fmtExclude = [ ./gen ./docs ]; }); }; + + # Nix-built container images for the integration tests, plus the + # per-test VM check factory. Linux-only (dockerTools + nixosTest). + integrationImages = import ./nix/images.nix { + inherit pkgs; + buildGoModule = pkgs.buildGo126Module; + tailscaleSrc = tailscale-head; + }; + + mkIntegrationCheck = { name, testFilter ? name, postgres ? false }: + pkgs.testers.nixosTest (import ./nix/tests/integration.nix { + inherit name testFilter pkgs postgres; + inherit (integrationImages) + headscaleImage tailscaleImage postgresImage tailscaleVersionImages; + testBin = pkgs.integration-test-bin; + # goSrc (not the full tree) so a check's result stays cached when + # only nix/CI/docs change — only real Go/integration changes (which + # also rebuild testBin) invalidate it. Maximises shared-cache reuse. + src = pkgs.headscale-go-src; + }); + + # One sqlite check per test (full matrix), plus a postgres variant for + # the postgres subset — exactly the sqlite+postgres jobs the generator + # produces for the GitHub workflow. + integrationChecks = + let + # Split-test entries (TestAutoApproveMultiNetwork/authkey-tag.*) carry + # / and * in their -test.run filter; sanitise those into a valid check + # name while keeping the raw filter for -test.run. + sanitize = builtins.replaceStrings [ "/" "." "*" ] [ "-" "" "" ]; + mkCheck = postgres: filter: + pkgs.lib.nameValuePair + "integration-${sanitize filter}${pkgs.lib.optionalString postgres "-pg"}" + (mkIntegrationCheck { name = sanitize filter; testFilter = filter; inherit postgres; }); + in + pkgs.lib.listToAttrs ( + map (mkCheck false) (import ./integration/tests.nix) + ++ map (mkCheck true) (import ./integration/postgres-tests.nix) + ); in { # `nix develop` @@ -259,6 +359,14 @@ cat go.mod | ${pkgs.ripgrep}/bin/rg "\t" | ${pkgs.ripgrep}/bin/rg -v indirect | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.findutils}/bin/xargs go get -u go mod tidy '') + + # Regenerate integration/tailscale-versions.json after a capver bump + # so the offline checks pin the tailscale images the suite requests. + # It is a //go:generate step in the integration package (runs with + # capver under `make generate`); this just scopes it to the pins. + (pkgs.writeShellScriptBin "update-integration-images" '' + exec go generate ./integration/... + '') ]; shellHook = '' @@ -272,6 +380,28 @@ inherit headscale; inherit headscale-docker; default = headscale; + } + // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { + headscale-integration-image = integrationImages.headscaleImage; + tailscale-integration-image = integrationImages.tailscaleImage; + postgres-integration-image = integrationImages.postgresImage; + inherit (pkgs) integration-test-bin; + + # All shared inputs every integration check pulls in: the test binary + # and every image. CI builds this once (a "prime" job) so it lands in + # the shared nix cache before the per-test matrix fans out — the + # matrix jobs then substitute it instead of each rebuilding. + integration-deps = pkgs.linkFarm "integration-deps" ( + [ + { name = "test-bin"; path = pkgs.integration-test-bin; } + { name = "headscale"; path = integrationImages.headscaleImage; } + { name = "tailscale-head"; path = integrationImages.tailscaleImage; } + { name = "postgres"; path = integrationImages.postgresImage; } + ] + ++ pkgs.lib.imap0 + (i: img: { name = "ts-version-${toString i}"; path = img; }) + integrationImages.tailscaleVersionImages + ); }; # `nix run` @@ -287,6 +417,10 @@ } # The Go build/test checks are gated to Linux: parts of the tree are # Linux-specific and the pure unit subset is validated by CI. - // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks; + // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks + # Integration checks only on x86_64-linux: nixosTest needs KVM and the + # pinned version images are amd64 (nix/tailscale-versions.nix). aarch64 + # would need arch-specific pins. + // pkgs.lib.optionalAttrs (system == "x86_64-linux") integrationChecks; }); } diff --git a/integration/excluded-tests.json b/integration/excluded-tests.json new file mode 100644 index 000000000..a9160c4da --- /dev/null +++ b/integration/excluded-tests.json @@ -0,0 +1,7 @@ +[ + "TestHASubnetRouterFailoverBothOfflineCablePull", + "TestHASubnetRouterFailoverDockerDisconnect", + "TestHASubnetRouterPingFailover", + "TestPingAllByIPPublicDERP", + "TestTailscaleRustAxum" +] diff --git a/integration/postgres-tests.nix b/integration/postgres-tests.nix new file mode 100644 index 000000000..68d9a2228 --- /dev/null +++ b/integration/postgres-tests.nix @@ -0,0 +1,8 @@ +# Generated by gh-action-integration-generator.go; do not edit. +[ + "TestACLAllowUserDst" + "TestPingAllByIP" + "TestEphemeral2006DeletedTooQuickly" + "TestPingAllByIPManyUpDown" + "TestSubnetRouterMultiNetwork" +] diff --git a/integration/tests.nix b/integration/tests.nix new file mode 100644 index 000000000..f2f2d1675 --- /dev/null +++ b/integration/tests.nix @@ -0,0 +1,156 @@ +# Generated by gh-action-integration-generator.go; do not edit. +[ + "TestACLHostsInNetMapTable" + "TestACLAllowUser80Dst" + "TestACLDenyAllPort80" + "TestACLAllowUserDst" + "TestACLAllowStarDst" + "TestACLNamedHostsCanReachBySubnet" + "TestACLNamedHostsCanReach" + "TestACLDevice1CanAccessDevice2" + "TestPolicyUpdateWhileRunningWithCLIInDatabase" + "TestACLAutogroupMember" + "TestACLAutogroupTagged" + "TestACLAutogroupSelf" + "TestACLPolicyPropagationOverTime" + "TestACLTagPropagation" + "TestACLTagPropagationPortSpecific" + "TestACLGroupWithUnknownUser" + "TestACLGroupAfterUserDeletion" + "TestACLGroupDeletionExactReproduction" + "TestACLDynamicUnknownUserAddition" + "TestACLDynamicUnknownUserRemoval" + "TestAPIAuthenticationBypass" + "TestAPIAuthenticationBypassCurl" + "TestRemoteCLIAuthenticationBypass" + "TestCLIWithConfigAuthenticationBypass" + "TestAuthKeyLogoutAndReloginSameUser" + "TestAuthKeyLogoutAndReloginNewUser" + "TestAuthKeyLogoutAndReloginSameUserExpiredKey" + "TestAuthKeyDeleteKey" + "TestAuthKeyLogoutAndReloginRoutesPreserved" + "TestOIDCAuthenticationPingAll" + "TestOIDCExpireNodesBasedOnTokenExpiry" + "TestOIDC024UserCreation" + "TestOIDCAuthenticationWithPKCE" + "TestOIDCReloginSameNodeNewUser" + "TestOIDCFollowUpUrl" + "TestOIDCMultipleOpenedLoginUrls" + "TestOIDCReloginSameNodeSameUser" + "TestOIDCExpiryAfterRestart" + "TestOIDCACLPolicyOnJoin" + "TestOIDCReloginSameUserRoutesPreserved" + "TestAuthWebFlowAuthenticationPingAll" + "TestAuthWebFlowLogoutAndReloginSameUser" + "TestAuthWebFlowLogoutAndReloginNewUser" + "TestApiKeyCommand" + "TestApiKeyCommandValidation" + "TestAuthCommandValidation" + "TestNodeCommand" + "TestNodeExpireCommand" + "TestNodeRenameCommand" + "TestPreAuthKeyCorrectUserLoggedInCommand" + "TestTaggedNodesCLIOutput" + "TestNodeExpireFlagsCommand" + "TestNodeCommandValidation" + "TestNodeTagCommand" + "TestNodeRouteCommands" + "TestNodeBackfillIPsCommand" + "TestPolicyCheckCommand" + "TestSSHTestsRejectFailingPolicy" + "TestPolicyCommand" + "TestPolicyBrokenConfigCommand" + "TestPreAuthKeyCommand" + "TestPreAuthKeyCommandWithoutExpiry" + "TestPreAuthKeyCommandReusableEphemeral" + "TestPreAuthKeyDeleteCommand" + "TestPreAuthKeyCommandValidation" + "TestServerInfoCommands" + "TestUserCommand" + "TestUserCreateCommand" + "TestUserCommandValidation" + "TestDERPVerifyEndpoint" + "TestResolveMagicDNS" + "TestResolveMagicDNSExtraRecordsPath" + "TestDERPServerScenario" + "TestDERPServerWebsocketScenario" + "TestPingAllByIP" + "TestEphemeral" + "TestEphemeralInAlternateTimezone" + "TestEphemeral2006DeletedTooQuickly" + "TestPingAllByHostname" + "TestTaildrop" + "TestUpdateHostnameFromClient" + "TestExpireNode" + "TestSetNodeExpiryInFuture" + "TestDisableNodeExpiry" + "TestNodeOnlineStatus" + "TestPingAllByIPManyUpDown" + "Test2118DeletingOnlineNodePanics" + "TestGrantCapRelay" + "TestGrantCapDrive" + "TestEnablingRoutes" + "TestHASubnetRouterFailover" + "TestSubnetRouteACL" + "TestEnablingExitRoutes" + "TestExitRoutesWithAutogroupInternetACL" + "TestSubnetRouterMultiNetwork" + "TestSubnetRouterMultiNetworkExitNode" + "TestAutoApproveMultiNetwork/authkey-tag.*" + "TestAutoApproveMultiNetwork/authkey-user.*" + "TestAutoApproveMultiNetwork/authkey-group.*" + "TestAutoApproveMultiNetwork/webauth-tag.*" + "TestAutoApproveMultiNetwork/webauth-user.*" + "TestAutoApproveMultiNetwork/webauth-group.*" + "TestSubnetRouteACLFiltering" + "TestGrantViaSubnetSteering" + "TestHASubnetRouterFailoverBothOffline" + "TestHeadscale" + "TestTailscaleNodesJoiningHeadcale" + "TestSSHOneUserToAll" + "TestSSHMultipleUsersAllToAll" + "TestSSHNoSSHConfigured" + "TestSSHIsBlockedInACL" + "TestSSHUserOnlyIsolation" + "TestSSHAutogroupSelf" + "TestSSHOneUserToOneCheckModeCLI" + "TestSSHOneUserToOneCheckModeOIDC" + "TestSSHCheckModeUnapprovedTimeout" + "TestSSHCheckModeCheckPeriodCLI" + "TestSSHCheckModeAutoApprove" + "TestSSHCheckModeSessionLossReDelegates" + "TestSSHCheckModeNegativeCLI" + "TestSSHLocalpart" + "TestTagsAuthKeyWithTagRequestDifferentTag" + "TestTagsAuthKeyWithTagNoAdvertiseFlag" + "TestTagsAuthKeyWithTagCannotAddViaCLI" + "TestTagsAuthKeyWithTagCannotChangeViaCLI" + "TestTagsAuthKeyWithTagAdminOverrideReauthPreserves" + "TestTagsAuthKeyWithTagCLICannotModifyAdminTags" + "TestTagsAuthKeyWithoutTagCannotRequestTags" + "TestTagsAuthKeyWithoutTagRegisterNoTags" + "TestTagsAuthKeyWithoutTagCannotAddViaCLI" + "TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset" + "TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise" + "TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag" + "TestTagsUserLoginOwnedTagAtRegistration" + "TestTagsUserLoginNonExistentTagAtRegistration" + "TestTagsUserLoginUnownedTagAtRegistration" + "TestTagsUserLoginAddTagViaCLIReauth" + "TestTagsUserLoginRemoveTagViaCLIReauth" + "TestTagsUserLoginCLINoOpAfterAdminAssignment" + "TestTagsUserLoginCLICannotRemoveAdminTags" + "TestTagsAuthKeyWithTagRequestNonExistentTag" + "TestTagsAuthKeyWithTagRequestUnownedTag" + "TestTagsAuthKeyWithoutTagRequestNonExistentTag" + "TestTagsAuthKeyWithoutTagRequestUnownedTag" + "TestTagsAdminAPICannotSetNonExistentTag" + "TestTagsAdminAPICanSetUnownedTag" + "TestTagsAdminAPICannotRemoveAllTags" + "TestTagsIssue2978ReproTagReplacement" + "TestTagsAdminAPICannotSetInvalidFormat" + "TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags" + "TestTagsAuthKeyWithoutUserInheritsTags" + "TestTagsAuthKeyWithoutUserRejectsAdvertisedTags" + "TestTagsAuthKeyConvertToUserViaCLIRegister" +] diff --git a/nix/tests/integration.nix b/nix/tests/integration.nix new file mode 100644 index 000000000..e96a861d9 --- /dev/null +++ b/nix/tests/integration.nix @@ -0,0 +1,157 @@ +# Per-test integration check: a NixOS VM that boots docker, loads the nix-built +# + pinned images (full tailscale version matrix), and runs the prebuilt +# integration test binary filtered to a single test. Hermetic equivalent of one +# `go run ./cmd/hi run ` job. `postgres = true` produces the postgres +# variant (HEADSCALE_INTEGRATION_POSTGRES=1) for the postgres subset. +# +# The suite runs *inside* a container named headscale-test-suite- with +# the docker socket mounted (docker-out-of-docker): scenario.go attaches that +# container to each test network, and IntegrationSkip gates on /.dockerenv. We +# reuse the headscale image as the runner base and bind-mount the host nix store +# so the test binary's closure resolves. +{ name +, testFilter ? name +, pkgs +, headscaleImage +, tailscaleImage +, postgresImage +, tailscaleVersionImages +, testBin +, src +, postgres ? false +}: +let + lib = pkgs.lib; + + # The test function name, dropping any "/subtest" filter on a split check, so + # the skip-guard below matches a whole-function skip without tripping on a + # legitimately-skipped subtest. + testFunc = builtins.head (lib.splitString "/" testFilter); + + # >=6 chars and DNS-safe: tsic/hsic derive container hostnames from the last + # 6 chars of the run ID (runID[len-6:]), which panics on shorter ids. + runID = "nixcheck"; + runnerName = "headscale-test-suite-${runID}"; + + # Images this check must have locally before the suite runs. Every test may + # fan out over the full tailscale version matrix (MustTestVersions); the head + # image doubles as the DERP server; the postgres variant adds postgres. + images = + [ headscaleImage tailscaleImage ] + ++ tailscaleVersionImages + ++ lib.optional postgres postgresImage; + + # Loading ~8 images dominates wall time; do it in parallel (they share layers, + # so docker dedups) and fail if any load fails. + loadImages = pkgs.writeShellScript "load-integration-images" '' + set -u + pids="" + ${lib.concatMapStringsSep "\n" (img: ''docker load -i ${img} & pids="$pids $!"'') images} + rc=0 + for p in $pids; do wait "$p" || rc=1; done + exit "$rc" + ''; + + pgEnv = lib.optionalString postgres + "-e HEADSCALE_INTEGRATION_POSTGRES=1 -e HEADSCALE_INTEGRATION_POSTGRES_IMAGE=postgres:16 "; +in +{ + name = "integration-${name}${lib.optionalString postgres "-pg"}"; + + nodes.machine = { ... }: { + virtualisation.docker.enable = true; + + # Match the integration CI: Docker 29's default containerd snapshotter + + # overlayfs regresses image load/exec for these images; the classic + # overlay2 graph driver is the known-good path. + virtualisation.docker.daemon.settings = { + storage-driver = "overlay2"; + features.containerd-snapshotter = false; + }; + + # tailscaled needs /dev/net/tun for its --tun=tsdev device. + boot.kernelModules = [ "tun" ]; + + # The VM's dhcpcd otherwise manages docker's bridges/veths: as containers + # and the cable-pull tests churn interfaces it deletes their routes and + # races libnetwork's gateway setup, surfacing as + # "API error (500): updating gateway endpoint: failed to set gateway: + # network is unreachable" on reconnect (TestHASubnetRouterFailover*Disconnect + # /CablePull) and intermittent container-startup flakes. Keep it off them. + networking.dhcpcd.denyInterfaces = [ "veth*" "br-*" "docker*" ]; + + # Trim the guest toward a tiny profile: this VM only needs to boot, run + # docker, and exec one test binary. Measured boot is 18.7s (1.7s kernel + + # 3.9s initrd + 13.1s userspace); the userspace critical chain is + # dhcpcd (6.1s) -> network-online.target -> docker.service (+3.5s), i.e. the + # remaining cost is functional (docker waits for the network), not fat to + # trim. documentation + timesyncd off shave the cheap, avoidable part. + # + # microvm.nix was evaluated against this and rejected: it still needs a full + # docker daemon + nested KVM, and boot is only ~10-15% of the 2-3 min + # image-load + test runtime, so a new flake input can't pay itself back. The + # one real lever left, dropping docker's network-online wait (~6s), sits + # right next to the dhcpcd interface races handled below and isn't worth + # destabilising the matrix for. + documentation.enable = false; + services.timesyncd.enable = false; + + # Sized for the full version matrix (a test can spawn 12+ tailscale + # containers + headscale + the runner) while still fitting a standard + # 4-core/16 GB GitHub runner. Disk-backed docker (not tmpfs) keeps RAM + # headroom on the runner; the slim source copy below is the bigger win. + virtualisation.memorySize = 8192; + virtualisation.cores = 4; + virtualisation.diskSize = 12288; + }; + + testScript = '' + start_all() + machine.wait_for_unit("docker.service") + + # Load all matrix images in parallel (store paths visible inside the VM). + machine.succeed("${loadImages}", timeout=600) + + # Writable copy of just the integration package (the suite writes + # control_logs/ under CWD and reads testdata relative to it). Only this + # subtree is needed at runtime — the repo-root build context is only used + # for image builds, which we bypass with prebuilt images. + machine.succeed("mkdir -p /tmp/hs && cp -r ${src}/integration /tmp/hs/integration && chmod -R u+w /tmp/hs") + + # hsic/tsic save artifacts (and tsic.Status writes _status.json) under + # /tmp/control; cmd/hi mounts this dir. Without it every Status() errors on + # the write and the suite never observes NeedsLogin. + machine.succeed("mkdir -p /tmp/control") + + # Run the suite inside a container (docker-out-of-docker), mirroring cmd/hi: + # - name matches scenario.go's testSuiteName so it can join test networks + # - docker socket mounted so sibling containers spawn on this daemon + # - host nix store mounted so the test binary's closure resolves + out = machine.succeed( + "docker run --rm --name ${runnerName} " + "-v /var/run/docker.sock:/var/run/docker.sock " + "-v /nix/store:/nix/store:ro " + "-v /tmp/hs:/tmp/hs -w /tmp/hs/integration " + "-v /tmp/control:/tmp/control " + "-e HEADSCALE_INTEGRATION_HEADSCALE_IMAGE=headscale:nix " + "-e HEADSCALE_INTEGRATION_TAILSCALE_IMAGE=tailscale-head:nix " + "-e HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE=tailscale-head:nix " + "-e HEADSCALE_INTEGRATION_DERPER_IMAGE=tailscale-head:nix " + "${pgEnv}" + "-e HEADSCALE_INTEGRATION_RUN_ID=${runID} " + "-e CI=1 " + # The nested-docker VM converges slower than a host docker daemon; widen + # the ScaledTimeout budgets (subnet-route convergence, HA cycles) past 2x. + "-e HEADSCALE_INTEGRATION_TIMEOUT_SCALE=4 " + "-e SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt " + "headscale:nix " + "${testBin}/bin/integration.test -test.run '^${testFilter}$' -test.v -test.timeout=20m", + timeout=1800, + ) + print(out) + + # A skipped or unmatched test still exits 0; guard against a false green. + assert "no tests to run" not in out, "filter ${testFilter} matched nothing" + assert "--- SKIP: ${testFunc} " not in out, "test ${testFunc} was skipped (not in container?)" + ''; +} From 4e5e538903b5b00728767d1a705fbf6c12031a9b Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 25 Jun 2026 08:07:29 +0000 Subject: [PATCH 5/7] ci: replace the integration workflow with the flake checks and a legacy pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI runs nix build .#checks..integration-* instead of the bespoke docker matrix — the generated checks are the matrix. The tests the hermetic VM can't run (real-internet DERP, docker cable-pull) run via cmd/hi in a slim legacy workflow, which also keeps cmd/hi and the integration Dockerfiles exercised. check-tests guards the pinned versions against capver drift. --- .github/workflows/check-tests.yaml | 18 +- .../workflows/integration-test-template.yml | 146 ------ .../workflows/test-integration-legacy.yaml | 75 +++ .github/workflows/test-integration.yaml | 433 ++---------------- 4 files changed, 132 insertions(+), 540 deletions(-) delete mode 100644 .github/workflows/integration-test-template.yml create mode 100644 .github/workflows/test-integration-legacy.yaml diff --git a/.github/workflows/check-tests.yaml b/.github/workflows/check-tests.yaml index b73011732..dd3637b22 100644 --- a/.github/workflows/check-tests.yaml +++ b/.github/workflows/check-tests.yaml @@ -37,9 +37,23 @@ jobs: if: steps.changed-files.outputs.files == 'true' run: | (cd .github/workflows && go generate) - git diff --exit-code .github/workflows/test-integration.yaml + git diff --exit-code integration/tests.nix integration/postgres-tests.nix integration/excluded-tests.json - name: Show missing tests if: failure() run: | - git diff .github/workflows/test-integration.yaml + git diff integration/tests.nix integration/postgres-tests.nix integration/excluded-tests.json + + # The pinned tailscale images must match the versions the suite requests + # (MustTestVersions); a capver bump that isn't mirrored here makes the + # offline integration checks fail at runtime. + - name: Check tailscale version pins match capver + if: steps.changed-files.outputs.files == 'true' + run: | + want=$(nix develop --command go run ./cmd/hi list-versions --set=must --exclude=head | tr ' ' '\n' | sort) + have=$(jq -r '.images | keys[]' integration/tailscale-versions.json | sort) + if [ "$want" != "$have" ]; then + echo "::error::integration/tailscale-versions.json is stale vs capver; run 'update-integration-images' in the dev shell" + diff <(echo "$want") <(echo "$have") || true + exit 1 + fi diff --git a/.github/workflows/integration-test-template.yml b/.github/workflows/integration-test-template.yml deleted file mode 100644 index 590185c96..000000000 --- a/.github/workflows/integration-test-template.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: Integration Test Template - -on: - workflow_call: - inputs: - test: - required: true - type: string - postgres_flag: - required: false - type: string - default: "" - database_name: - required: true - type: string - -jobs: - test: - runs-on: ubuntu-24.04-arm - env: - # Github does not allow us to access secrets in pull requests, - # so this env var is used to check if we have the secret or not. - # If we have the secrets, meaning we are running on push in a fork, - # there might be secrets available for more debugging. - # If TS_OAUTH_CLIENT_ID and TS_OAUTH_SECRET is set, then the job - # will join a debug tailscale network, set up SSH and a tmux session. - # The SSH will be configured to use the SSH key of the Github user - # that triggered the build. - HAS_TAILSCALE_SECRET: ${{ secrets.TS_OAUTH_CLIENT_ID }} - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 2 - - name: Tailscale - if: ${{ env.HAS_TAILSCALE_SECRET }} - uses: tailscale/github-action@a392da0a182bba0e9613b6243ebd69529b1878aa # v4.1.0 - with: - oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} - oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} - tags: tag:gh - - name: Setup SSH server for Actor - if: ${{ env.HAS_TAILSCALE_SECRET }} - uses: alexellis/setup-sshd-actor@master - - name: Download headscale image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: headscale-image - path: /tmp/artifacts - - name: Download tailscale HEAD image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: tailscale-head-image - path: /tmp/artifacts - - name: Download tailscale released images - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: tailscale-released-images - path: /tmp/artifacts - - name: Download hi binary - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: hi-binary - path: /tmp/artifacts - - name: Download Go cache - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: go-cache - path: /tmp/artifacts - - name: Download postgres image - if: ${{ inputs.postgres_flag == '--postgres=1' }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: postgres-image - path: /tmp/artifacts - - name: Force overlay2 storage driver - run: | - sudo mkdir -p /etc/docker - echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker version - - name: Load br_netfilter for in-cluster service routing - if: inputs.test == 'TestK8sOperator' - # TestK8sOperator runs k3s in a container; without br_netfilter on the - # host, bridged pod-to-pod traffic skips kube-proxy's ClusterIP DNAT and - # in-cluster DNS (kube-dns) is unreachable. The module cannot be loaded - # from inside the unprivileged-module rancher/k3s image, so load it here. - run: sudo modprobe br_netfilter - - name: Login to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - if: env.DOCKERHUB_USERNAME != '' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: Load Docker images, Go cache, and prepare binary - run: | - gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load - gunzip -c /tmp/artifacts/tailscale-head-image.tar.gz | docker load - gunzip -c /tmp/artifacts/tailscale-released-images.tar.gz | docker load - if [ -f /tmp/artifacts/postgres-image.tar.gz ]; then - gunzip -c /tmp/artifacts/postgres-image.tar.gz | docker load - fi - chmod +x /tmp/artifacts/hi - docker images - # Extract Go cache to host directories for bind mounting - mkdir -p /tmp/go-cache - tar -xzf /tmp/artifacts/go-cache.tar.gz -C /tmp/go-cache - ls -la /tmp/go-cache/ /tmp/go-cache/.cache/ - - name: Run Integration Test - env: - HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:${{ github.sha }} - HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:${{ github.sha }} - HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }} - HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go - HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build - # Mirror the docker/login-action secrets into env so the - # dockertestutil.Credentials resolver picks them up directly - # (otherwise it falls back to parsing ~/.docker/config.json, - # which works but is one step further from the source). - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \ - --timeout=120m \ - ${{ inputs.postgres_flag }} - # Sanitize test name for artifact upload (replace invalid characters: " : < > | * ? \ / with -) - - name: Sanitize test name for artifacts - if: always() - id: sanitize - run: echo "name=${TEST_NAME//[\":<>|*?\\\/]/-}" >> $GITHUB_OUTPUT - env: - TEST_NAME: ${{ inputs.test }} - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - if: always() - with: - name: ${{ inputs.database_name }}-${{ steps.sanitize.outputs.name }}-logs - path: "control_logs/*/*.log" - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - if: always() - with: - name: ${{ inputs.database_name }}-${{ steps.sanitize.outputs.name }}-artifacts - path: control_logs/ - - name: Setup a blocking tmux session - if: ${{ env.HAS_TAILSCALE_SECRET }} - uses: alexellis/block-with-tmux-action@master diff --git a/.github/workflows/test-integration-legacy.yaml b/.github/workflows/test-integration-legacy.yaml new file mode 100644 index 000000000..5a56142e6 --- /dev/null +++ b/.github/workflows/test-integration-legacy.yaml @@ -0,0 +1,75 @@ +name: integration-legacy + +on: + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +# A small number of integration tests cannot run as hermetic NixOS-VM checks +# because they need Tailscale's public DERP relays (real internet) — see +# excludedFromNixChecks in gh-action-integration-generator.go. They run here the +# legacy way, via cmd/hi against Dockerfile-built images, which also keeps the +# cmd/hi path and the integration Dockerfiles exercised in CI. The matrix is the +# generated integration/excluded-tests.json so this list has a single source. +jobs: + list: + runs-on: ubuntu-latest + outputs: + tests: ${{ steps.list.outputs.tests }} + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - id: list + run: echo "tests=$(jq -c . integration/excluded-tests.json)" >> "$GITHUB_OUTPUT" + + test: + needs: list + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test: ${{ fromJSON(needs.list.outputs.tests) }} + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main + - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main + + # Docker 29's default containerd snapshotter regresses these images; the + # classic overlay2 graph driver is the known-good path (mirrors the VM checks). + - name: Force overlay2 storage driver + run: | + sudo mkdir -p /etc/docker + echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + # cmd/hi requires prebuilt head images in CI (util.IsCI); released versions + # pull from the registry and tailscale-rs is built inline by tsric. + # Dockerfile.integration-ci COPYs a prebuilt headscale binary from the + # build context, so build it first (tailscale-HEAD builds from source). + - name: Build headscale + tailscale-HEAD images + run: | + nix develop --command bash -c 'CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale' + docker build --file Dockerfile.integration-ci --tag headscale:ci . + docker build --file Dockerfile.tailscale-HEAD --tag tailscale-head:ci . + + - name: Run ${{ matrix.test }} + env: + HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:ci + HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:ci + run: nix develop --command go run ./cmd/hi run "^${{ matrix.test }}$" --timeout=900s diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index 974e55233..3b8e858ce 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -1,406 +1,55 @@ name: integration -# To debug locally on a branch, and when needing secrets -# change this to include `push` so the build is ran on -# the main repository. -on: [pull_request] + +on: + pull_request: + branches: + - main + concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -defaults: - run: - shell: nix develop --fallback --command bash -e {0} + +permissions: + contents: read + +# Each test is a NixOS-VM flake check (full tailscale version matrix, sqlite +# plus a postgres variant) — one job per check, derived from the checks the +# flake exposes. The fileset-filtered, content-addressed checks hit the hestia +# cache when their inputs are unchanged. A prime job builds the shared images + +# test binary once so the matrix substitutes them instead of each rebuilding. jobs: - # build: Builds binaries and Docker images once, uploads as artifacts for reuse. - # build-postgres: Pulls postgres image separately to avoid Docker Hub rate limits. - # build-tailscale-released: Pre-pulls released Tailscale images from ghcr.io - # so fork PRs (no DOCKERHUB_USERNAME secret) don't hit Docker Hub rate - # limits at test time. - # sqlite: Runs all integration tests with SQLite backend. - # postgres: Runs a subset of tests with PostgreSQL to verify database compatibility. - build: - runs-on: ubuntu-24.04-arm + list: + runs-on: ubuntu-latest outputs: - files-changed: ${{ steps.changed-files.outputs.files }} - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 2 - - name: Get changed files - id: changed-files - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - with: - filters: | - files: - - '*.nix' - - 'go.*' - - '**/*.go' - - 'integration/**' - - 'config-example.yaml' - - '.github/workflows/test-integration.yaml' - - '.github/workflows/integration-test-template.yml' - - 'Dockerfile.*' - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - if: steps.changed-files.outputs.files == 'true' - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - if: steps.changed-files.outputs.files == 'true' - - name: Build binaries and warm Go cache - if: steps.changed-files.outputs.files == 'true' - run: | - # Build all Go binaries in one nix shell to maximize cache reuse - go build -o hi ./cmd/hi - CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale - # Build integration test binary to warm the cache with all dependencies - go test -c ./integration -o /dev/null 2>/dev/null || true - - name: Upload hi binary - if: steps.changed-files.outputs.files == 'true' - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: hi-binary - path: hi - retention-days: 10 - - name: Package Go cache - if: steps.changed-files.outputs.files == 'true' - run: | - # Package Go module cache and build cache - tar -czf go-cache.tar.gz -C ~ go .cache/go-build - - name: Upload Go cache - if: steps.changed-files.outputs.files == 'true' - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: go-cache - path: go-cache.tar.gz - retention-days: 10 - - name: Force overlay2 storage driver - if: steps.changed-files.outputs.files == 'true' - run: | - # Docker 29 runner images default to overlayfs, which breaks - # docker build via Go SDK libraries and docker save/load - # tarball formats. overlay2 is the long-standing default. - # https://github.com/actions/runner-images/issues/13474 - sudo mkdir -p /etc/docker - echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker version - - name: Login to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - if: env.DOCKERHUB_USERNAME != '' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: Build headscale image - if: steps.changed-files.outputs.files == 'true' - run: | - docker build \ - --file Dockerfile.integration-ci \ - --tag headscale:${{ github.sha }} \ - . - docker save headscale:${{ github.sha }} | gzip > headscale-image.tar.gz - - name: Build tailscale HEAD image - if: steps.changed-files.outputs.files == 'true' - run: | - docker build \ - --file Dockerfile.tailscale-HEAD \ - --tag tailscale-head:${{ github.sha }} \ - . - docker save tailscale-head:${{ github.sha }} | gzip > tailscale-head-image.tar.gz - - name: Upload headscale image - if: steps.changed-files.outputs.files == 'true' - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: headscale-image - path: headscale-image.tar.gz - retention-days: 10 - - name: Upload tailscale HEAD image - if: steps.changed-files.outputs.files == 'true' - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: tailscale-head-image - path: tailscale-head-image.tar.gz - retention-days: 10 - build-postgres: - runs-on: ubuntu-24.04-arm - needs: build - if: needs.build.outputs.files-changed == 'true' - steps: - - name: Force overlay2 storage driver - shell: bash - run: | - sudo mkdir -p /etc/docker - echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker version - - name: Login to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - if: env.DOCKERHUB_USERNAME != '' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: Pull and save postgres image - shell: bash - run: | - docker pull postgres:latest - docker tag postgres:latest postgres:${{ github.sha }} - docker save postgres:${{ github.sha }} | gzip > postgres-image.tar.gz - - name: Upload postgres image - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: postgres-image - path: postgres-image.tar.gz - retention-days: 10 - build-tailscale-released: - runs-on: ubuntu-24.04-arm - needs: build - if: needs.build.outputs.files-changed == 'true' + checks: ${{ steps.list.outputs.checks }} steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: Force overlay2 storage driver + - id: list run: | - sudo mkdir -p /etc/docker - echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker version - - name: Login to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - if: env.DOCKERHUB_USERNAME != '' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: List Tailscale versions to pre-pull - id: versions - run: | - versions=$(go run ./cmd/hi list-versions --set=must --exclude=head) - echo "versions=${versions}" >> "$GITHUB_OUTPUT" - echo "Pre-pulling: ${versions}" - - name: Pull Tailscale images - run: | - # Releases come from ghcr.io (anonymous, unmetered). The - # "unstable" floating tag on ghcr.io has been stale since 2022, - # so it still needs to come from Docker Hub. xargs -P 0 fans - # out one process per tag and returns non-zero if any pull - # fails. - refs="" - for v in ${{ steps.versions.outputs.versions }}; do - if [ "${v}" = "unstable" ]; then - refs="${refs} tailscale/tailscale:${v}" - else - refs="${refs} ghcr.io/tailscale/tailscale:${v}" - fi - done - echo "${refs}" | tr ' ' '\n' | grep -v '^$' \ - | xargs -P 0 -I{} docker pull "{}" - echo "REFS=${refs}" >> "$GITHUB_ENV" - - name: Save Tailscale images to tarball - run: | - # Single docker save with all refs: one consistent snapshot, no - # parallel-daemon race. - docker save ${REFS} | gzip > tailscale-released-images.tar.gz - ls -lh tailscale-released-images.tar.gz - - name: Upload Tailscale released images - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: tailscale-released-images - path: tailscale-released-images.tar.gz - retention-days: 10 - sqlite: - needs: [build, build-tailscale-released] - if: needs.build.outputs.files-changed == 'true' + checks=$(nix eval --json .#checks.x86_64-linux --apply builtins.attrNames \ + | jq -c '[.[] | select(startswith("integration-"))]') + echo "checks=$checks" >> "$GITHUB_OUTPUT" + + prime: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main + - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main + - run: nix build -L --fallback .#packages.x86_64-linux.integration-deps + + integration: + needs: [list, prime] + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - test: - - TestACLHostsInNetMapTable - - TestACLAllowUser80Dst - - TestACLDenyAllPort80 - - TestACLAllowUserDst - - TestACLAllowStarDst - - TestACLNamedHostsCanReachBySubnet - - TestACLNamedHostsCanReach - - TestACLDevice1CanAccessDevice2 - - TestPolicyUpdateWhileRunningWithCLIInDatabase - - TestACLAutogroupMember - - TestACLAutogroupTagged - - TestACLAutogroupSelf - - TestACLPolicyPropagationOverTime - - TestACLTagPropagation - - TestACLTagPropagationPortSpecific - - TestACLGroupWithUnknownUser - - TestACLGroupAfterUserDeletion - - TestACLGroupDeletionExactReproduction - - TestACLDynamicUnknownUserAddition - - TestACLDynamicUnknownUserRemoval - - TestAPIAuthenticationBypass - - TestAPIAuthenticationBypassCurl - - TestRemoteCLIAuthenticationBypass - - TestCLIWithConfigAuthenticationBypass - - TestAuthKeyLogoutAndReloginSameUser - - TestAuthKeyLogoutAndReloginNewUser - - TestAuthKeyLogoutAndReloginSameUserExpiredKey - - TestAuthKeyDeleteKey - - TestAuthKeyLogoutAndReloginRoutesPreserved - - TestOIDCAuthenticationPingAll - - TestOIDCExpireNodesBasedOnTokenExpiry - - TestOIDC024UserCreation - - TestOIDCAuthenticationWithPKCE - - TestOIDCReloginSameNodeNewUser - - TestOIDCFollowUpUrl - - TestOIDCMultipleOpenedLoginUrls - - TestOIDCReloginSameNodeSameUser - - TestOIDCExpiryAfterRestart - - TestOIDCACLPolicyOnJoin - - TestOIDCReloginSameUserRoutesPreserved - - TestAuthWebFlowAuthenticationPingAll - - TestAuthWebFlowLogoutAndReloginSameUser - - TestAuthWebFlowLogoutAndReloginNewUser - - TestApiKeyCommand - - TestApiKeyCommandValidation - - TestAuthCommandValidation - - TestNodeCommand - - TestNodeExpireCommand - - TestNodeRenameCommand - - TestPreAuthKeyCorrectUserLoggedInCommand - - TestTaggedNodesCLIOutput - - TestNodeExpireFlagsCommand - - TestNodeCommandValidation - - TestNodeTagCommand - - TestNodeRouteCommands - - TestNodeBackfillIPsCommand - - TestOAuthClientCommand - - TestOAuthClientCommandValidation - - TestPolicyCheckCommand - - TestSSHTestsRejectFailingPolicy - - TestPolicyCommand - - TestPolicyBrokenConfigCommand - - TestPreAuthKeyCommand - - TestPreAuthKeyCommandWithoutExpiry - - TestPreAuthKeyCommandReusableEphemeral - - TestPreAuthKeyDeleteCommand - - TestPreAuthKeyCommandValidation - - TestServerInfoCommands - - TestUserCommand - - TestUserCreateCommand - - TestUserCommandValidation - - TestDERPVerifyEndpoint - - TestResolveMagicDNS - - TestResolveMagicDNSExtraRecordsPath - - TestDERPServerScenario - - TestDERPServerWebsocketScenario - - TestPingAllByIP - - TestPingAllByIPPublicDERP - - TestEphemeral - - TestEphemeralInAlternateTimezone - - TestEphemeral2006DeletedTooQuickly - - TestPingAllByHostname - - TestTaildrop - - TestUpdateHostnameFromClient - - TestExpireNode - - TestSetNodeExpiryInFuture - - TestDisableNodeExpiry - - TestNodeOnlineStatus - - TestPingAllByIPManyUpDown - - Test2118DeletingOnlineNodePanics - - TestGrantCapRelay - - TestGrantCapDrive - - TestK8sOperator - - TestEnablingRoutes - - TestHASubnetRouterFailover - - TestSubnetRouteACL - - TestEnablingExitRoutes - - TestExitRoutesWithAutogroupInternetACL - - TestSubnetRouterMultiNetwork - - TestSubnetRouterMultiNetworkExitNode - - TestAutoApproveMultiNetwork/authkey-tag.* - - TestAutoApproveMultiNetwork/authkey-user.* - - TestAutoApproveMultiNetwork/authkey-group.* - - TestAutoApproveMultiNetwork/webauth-tag.* - - TestAutoApproveMultiNetwork/webauth-user.* - - TestAutoApproveMultiNetwork/webauth-group.* - - TestSubnetRouteACLFiltering - - TestGrantViaSubnetSteering - - TestHASubnetRouterPingFailover - - TestHASubnetRouterFailoverBothOffline - - TestHASubnetRouterFailoverBothOfflineCablePull - - TestHASubnetRouterFailoverDockerDisconnect - - TestHeadscale - - TestTailscaleNodesJoiningHeadcale - - TestSSHOneUserToAll - - TestSSHMultipleUsersAllToAll - - TestSSHNoSSHConfigured - - TestSSHIsBlockedInACL - - TestSSHUserOnlyIsolation - - TestSSHAutogroupSelf - - TestSSHOneUserToOneCheckModeCLI - - TestSSHOneUserToOneCheckModeOIDC - - TestSSHCheckModeUnapprovedTimeout - - TestSSHCheckModeCheckPeriodCLI - - TestSSHCheckModeAutoApprove - - TestSSHCheckModeSessionLossReDelegates - - TestSSHCheckModeNegativeCLI - - TestSSHLocalpart - - TestTagsAuthKeyWithTagRequestDifferentTag - - TestTagsAuthKeyWithTagNoAdvertiseFlag - - TestTagsAuthKeyWithTagCannotAddViaCLI - - TestTagsAuthKeyWithTagCannotChangeViaCLI - - TestTagsAuthKeyWithTagAdminOverrideReauthPreserves - - TestTagsAuthKeyWithTagCLICannotModifyAdminTags - - TestTagsAuthKeyWithoutTagCannotRequestTags - - TestTagsAuthKeyWithoutTagRegisterNoTags - - TestTagsAuthKeyWithoutTagCannotAddViaCLI - - TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset - - TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise - - TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag - - TestTagsUserLoginOwnedTagAtRegistration - - TestTagsUserLoginNonExistentTagAtRegistration - - TestTagsUserLoginUnownedTagAtRegistration - - TestTagsUserLoginAddTagViaCLIReauth - - TestTagsUserLoginRemoveTagViaCLIReauth - - TestTagsUserLoginCLINoOpAfterAdminAssignment - - TestTagsUserLoginCLICannotRemoveAdminTags - - TestTagsAuthKeyWithTagRequestNonExistentTag - - TestTagsAuthKeyWithTagRequestUnownedTag - - TestTagsAuthKeyWithoutTagRequestNonExistentTag - - TestTagsAuthKeyWithoutTagRequestUnownedTag - - TestTagsAdminAPICannotSetNonExistentTag - - TestTagsAdminAPICanSetUnownedTag - - TestTagsAdminAPICannotRemoveAllTags - - TestTagsIssue2978ReproTagReplacement - - TestTagsAdminAPICannotSetInvalidFormat - - TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags - - TestTagsAuthKeyWithoutUserInheritsTags - - TestTagsAuthKeyWithoutUserRejectsAdvertisedTags - - TestTagsAuthKeyConvertToUserViaCLIRegister - - TestTailscaleRustAxum - uses: ./.github/workflows/integration-test-template.yml - secrets: inherit - with: - test: ${{ matrix.test }} - postgres_flag: "--postgres=0" - database_name: "sqlite" - postgres: - needs: [build, build-postgres, build-tailscale-released] - if: needs.build.outputs.files-changed == 'true' - strategy: - fail-fast: false - matrix: - test: - - TestACLAllowUserDst - - TestPingAllByIP - - TestEphemeral2006DeletedTooQuickly - - TestPingAllByIPManyUpDown - - TestSubnetRouterMultiNetwork - uses: ./.github/workflows/integration-test-template.yml - secrets: inherit - with: - test: ${{ matrix.test }} - postgres_flag: "--postgres=1" - database_name: "postgres" + check: ${{ fromJson(needs.list.outputs.checks) }} + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main + - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main + - name: ${{ matrix.check }} + run: nix build -L --fallback .#checks.x86_64-linux.${{ matrix.check }} From 359127a800457fec4c439977dfc1ff47f4890ae8 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 1 Jul 2026 12:09:11 +0000 Subject: [PATCH 6/7] ci: remove the CI jobs garnix now runs as flake checks Garnix builds every flake check and package directly, so the GitHub Actions wrappers around them are redundant: - nix-checks checks.{build,gotest,golangci-lint,formatting} - nix-module-test checks.headscale (NixOS module test) - test-integration the integration-* NixOS-VM check matrix - build packages.headscale (nix build) Kept are the pipelines garnix cannot reproduce: servertest (excluded from the flake), integration-legacy (needs real DERP, so it cannot run hermetically), and the check-generated / check-tests generators that keep the nix integration matrix and protobuf output in sync. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01WxukxKem3HU4i5CqNrvvdj --- .github/workflows/build.yml | 95 ------------------------- .github/workflows/nix-checks.yml | 56 --------------- .github/workflows/nix-module-test.yml | 51 ------------- .github/workflows/test-integration.yaml | 55 -------------- integration/README.md | 15 ++-- 5 files changed, 9 insertions(+), 263 deletions(-) delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/nix-checks.yml delete mode 100644 .github/workflows/nix-module-test.yml delete mode 100644 .github/workflows/test-integration.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 81246706c..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Build - -on: - push: - branches: - - main - pull_request: - -concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -defaults: - run: - shell: nix develop --fallback --command bash -e {0} - -jobs: - build-nix: - runs-on: ubuntu-latest - permissions: write-all - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 2 - - name: Get changed files - id: changed-files - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - with: - filters: | - files: - - '*.nix' - - 'go.*' - - '**/*.go' - - 'integration_test/' - - 'config-example.yaml' - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - if: steps.changed-files.outputs.files == 'true' - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - if: steps.changed-files.outputs.files == 'true' - - - name: Check vendor hash - id: vendorhash - if: steps.changed-files.outputs.files == 'true' - run: | - go run ./cmd/vendorhash check | tee check-result - { - grep '^expected_sri=' check-result || true - grep '^actual_sri=' check-result || true - } >> "$GITHUB_OUTPUT" - - - name: Vendor hash diverging - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - if: failure() && steps.vendorhash.outcome == 'failure' - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - github.rest.pulls.createReviewComment({ - pull_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'Vendor hash in `flakehashes.json` is stale (was `${{ steps.vendorhash.outputs.expected_sri }}`, should be `${{ steps.vendorhash.outputs.actual_sri }}`). Run `go run ./cmd/vendorhash update` and commit the result.' - }) - - - name: Run nix build - if: steps.changed-files.outputs.files == 'true' - run: nix build --fallback - - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - if: steps.changed-files.outputs.files == 'true' - with: - name: headscale-linux - path: result/bin/headscale - build-cross: - runs-on: ubuntu-latest - strategy: - matrix: - env: - - "GOARCH=arm64 GOOS=linux" - - "GOARCH=amd64 GOOS=linux" - - "GOARCH=arm64 GOOS=darwin" - - "GOARCH=amd64 GOOS=darwin" - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - - name: Run go cross compile - env: - CGO_ENABLED: 0 - run: env ${{ matrix.env }} go build -o "headscale" - ./cmd/headscale - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: "headscale-${{ matrix.env }}" - path: "headscale" diff --git a/.github/workflows/nix-checks.yml b/.github/workflows/nix-checks.yml deleted file mode 100644 index 03c4b8e8b..000000000 --- a/.github/workflows/nix-checks.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Nix Flake Checks - -on: - push: - branches: - - main - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -# Each job only runs `nix build .#checks..`; the check logic lives -# in flake.nix via the flake-checks library. The fileset-filtered checks hit the -# hestia cache when their inputs are unchanged, so no changed-files gating. -permissions: - contents: read - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: build - run: nix build -L .#checks.x86_64-linux.build - - gotest: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: gotest - run: nix build -L .#checks.x86_64-linux.gotest - - golangci-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: golangci-lint - run: nix build -L .#checks.x86_64-linux.golangci-lint - - formatting: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: formatting - run: nix build -L .#checks.x86_64-linux.formatting diff --git a/.github/workflows/nix-module-test.yml b/.github/workflows/nix-module-test.yml deleted file mode 100644 index 1a7fbcc85..000000000 --- a/.github/workflows/nix-module-test.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: NixOS Module Tests - -on: - push: - branches: - - main - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - nix-module-check: - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 2 - - - name: Get changed files - id: changed-files - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - with: - filters: | - nix: - - 'nix/**' - - 'flake.nix' - - 'flake.lock' - go: - - 'go.*' - - '**/*.go' - - 'cmd/**' - - 'hscontrol/**' - - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true' - - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true' - - - name: Run NixOS module tests - if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true' - run: | - echo "Running NixOS module integration test..." - nix build .#checks.x86_64-linux.headscale -L --fallback diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml deleted file mode 100644 index 3b8e858ce..000000000 --- a/.github/workflows/test-integration.yaml +++ /dev/null @@ -1,55 +0,0 @@ -name: integration - -on: - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -# Each test is a NixOS-VM flake check (full tailscale version matrix, sqlite -# plus a postgres variant) — one job per check, derived from the checks the -# flake exposes. The fileset-filtered, content-addressed checks hit the hestia -# cache when their inputs are unchanged. A prime job builds the shared images + -# test binary once so the matrix substitutes them instead of each rebuilding. -jobs: - list: - runs-on: ubuntu-latest - outputs: - checks: ${{ steps.list.outputs.checks }} - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - id: list - run: | - checks=$(nix eval --json .#checks.x86_64-linux --apply builtins.attrNames \ - | jq -c '[.[] | select(startswith("integration-"))]') - echo "checks=$checks" >> "$GITHUB_OUTPUT" - - prime: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - run: nix build -L --fallback .#packages.x86_64-linux.integration-deps - - integration: - needs: [list, prime] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - check: ${{ fromJson(needs.list.outputs.checks) }} - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: ${{ matrix.check }} - run: nix build -L --fallback .#checks.x86_64-linux.${{ matrix.check }} diff --git a/integration/README.md b/integration/README.md index 5511a113f..ce180246d 100644 --- a/integration/README.md +++ b/integration/README.md @@ -21,16 +21,19 @@ go run ./cmd/hi doctor go run ./cmd/hi run "TestPingAllByIP" ``` -Alternatively, [`act`](https://github.com/nektos/act) runs the GitHub -Actions workflow locally: +In CI each test also runs as its own hermetic NixOS-VM flake check. Run +one locally exactly the way CI does: ```bash -act pull_request -W .github/workflows/test-integration.yaml +nix build .#checks.x86_64-linux.integration-TestPingAllByIP ``` -Each test runs as a separate workflow on GitHub Actions. To add a new -test, run `go generate` inside `../cmd/gh-action-integration-generator/` -and commit the generated workflow file. +The check matrix is generated: after adding or renaming a test, run `go +generate` in `.github/workflows/` to refresh `integration/tests.nix`, +`integration/postgres-tests.nix`, and `integration/excluded-tests.json`, +then commit the result. Tests that need the public internet (real DERP) +cannot run hermetically; they are listed in `excluded-tests.json` and +run via the `integration-legacy` workflow instead. ## Framework overview From a4a3633670194625a3652352a698bf55596ea942 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 1 Jul 2026 12:15:31 +0000 Subject: [PATCH 7/7] integration: refresh the nix test matrix for the rebased main Rebasing onto main pulled in TestK8sOperator and the OAuth client CLI tests; regenerate integration/tests.nix so the NixOS-VM check matrix runs them too. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01WxukxKem3HU4i5CqNrvvdj --- integration/tests.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration/tests.nix b/integration/tests.nix index f2f2d1675..5b14c095d 100644 --- a/integration/tests.nix +++ b/integration/tests.nix @@ -56,6 +56,8 @@ "TestNodeTagCommand" "TestNodeRouteCommands" "TestNodeBackfillIPsCommand" + "TestOAuthClientCommand" + "TestOAuthClientCommandValidation" "TestPolicyCheckCommand" "TestSSHTestsRejectFailingPolicy" "TestPolicyCommand" @@ -89,6 +91,7 @@ "Test2118DeletingOnlineNodePanics" "TestGrantCapRelay" "TestGrantCapDrive" + "TestK8sOperator" "TestEnablingRoutes" "TestHASubnetRouterFailover" "TestSubnetRouteACL"