Mirrors/headscale
1
0
Fork 0
mirror of https://github.com/juanfont/headscale.git synced 2026-07-20 18:09:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Merge a4a3633670 into 518b497be9

Browse source
This commit is contained in:
Kristoffer Dalby 2026-07-01 12:15:37 +00:00 • committed by GitHub
commit 29bcd4e5c2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 1536 additions and 992 deletions
Download patch file Download diff file Expand all files Collapse all files

95
.github/workflows/build.yml vendored
View file

@ -1,95 +0,0 @@
name: Build
on:
push:
branches:
- main
pull_request:
concurrency:
group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
cancel-in-progress: true
defaults:
run:
shell: nix develop --fallback --command bash -e {0}
jobs:
build-nix:
runs-on: ubuntu-latest
permissions: write-all
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
with:
filters: |
files:
- '*.nix'
- 'go.*'
- '**/*.go'
- 'integration_test/'
- 'config-example.yaml'
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
if: steps.changed-files.outputs.files == 'true'
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
if: steps.changed-files.outputs.files == 'true'
- name: Check vendor hash
id: vendorhash
if: steps.changed-files.outputs.files == 'true'
run: |
go run ./cmd/vendorhash check | tee check-result
{
grep '^expected_sri=' check-result || true
grep '^actual_sri=' check-result || true
} >> "$GITHUB_OUTPUT"
- name: Vendor hash diverging
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
if: failure() && steps.vendorhash.outcome == 'failure'
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
github.rest.pulls.createReviewComment({
pull_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: 'Vendor hash in `flakehashes.json` is stale (was `${{ steps.vendorhash.outputs.expected_sri }}`, should be `${{ steps.vendorhash.outputs.actual_sri }}`). Run `go run ./cmd/vendorhash update` and commit the result.'
})
- name: Run nix build
if: steps.changed-files.outputs.files == 'true'
run: nix build --fallback
- uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
if: steps.changed-files.outputs.files == 'true'
with:
name: headscale-linux
path: result/bin/headscale
build-cross:
runs-on: ubuntu-latest
strategy:
matrix:
env:
- "GOARCH=arm64 GOOS=linux"
- "GOARCH=amd64 GOOS=linux"
- "GOARCH=arm64 GOOS=darwin"
- "GOARCH=amd64 GOOS=darwin"
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
- name: Run go cross compile
env:
CGO_ENABLED: 0
run: env ${{ matrix.env }} go build -o "headscale"
./cmd/headscale
- uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: "headscale-${{ matrix.env }}"
path: "headscale"

18
.github/workflows/check-tests.yaml vendored
View file

@ -37,9 +37,23 @@ jobs:
if: steps.changed-files.outputs.files == 'true'
run: |
(cd .github/workflows && go generate)
git diff --exit-code .github/workflows/test-integration.yaml
git diff --exit-code integration/tests.nix integration/postgres-tests.nix integration/excluded-tests.json
- name: Show missing tests
if: failure()
run: |
git diff .github/workflows/test-integration.yaml
git diff integration/tests.nix integration/postgres-tests.nix integration/excluded-tests.json
# The pinned tailscale images must match the versions the suite requests
# (MustTestVersions); a capver bump that isn't mirrored here makes the
# offline integration checks fail at runtime.
- name: Check tailscale version pins match capver
if: steps.changed-files.outputs.files == 'true'
run: |
want=$(nix develop --command go run ./cmd/hi list-versions --set=must --exclude=head | tr ' ' '\n' | sort)
have=$(jq -r '.images | keys[]' integration/tailscale-versions.json | sort)
if [ "$want" != "$have" ]; then
echo "::error::integration/tailscale-versions.json is stale vs capver; run 'update-integration-images' in the dev shell"
diff <(echo "$want") <(echo "$have") || true
exit 1
fi

167
.github/workflows/gh-action-integration-generator.go vendored
View file

@ -4,30 +4,49 @@ package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"slices"
"strings"
)
// testsToSplit defines tests that should be split into multiple CI jobs.
// Key is the test function name, value is a list of subtest prefixes.
// Each prefix becomes a separate CI job as "TestName/prefix".
//
// Example: [TestAutoApproveMultiNetwork] has subtests like:
// - TestAutoApproveMultiNetwork/authkey-tag-advertiseduringup-false-pol-database
// - TestAutoApproveMultiNetwork/webauth-user-advertiseduringup-true-pol-file
//
// Splitting by approver type (tag, user, group) creates 6 CI jobs with 4 tests each:
// - TestAutoApproveMultiNetwork/authkey-tag.* (4 tests)
// - TestAutoApproveMultiNetwork/authkey-user.* (4 tests)
// - TestAutoApproveMultiNetwork/authkey-group.* (4 tests)
// - TestAutoApproveMultiNetwork/webauth-tag.* (4 tests)
// - TestAutoApproveMultiNetwork/webauth-user.* (4 tests)
// - TestAutoApproveMultiNetwork/webauth-group.* (4 tests)
//
// This reduces load per CI job (4 tests instead of 12) to avoid infrastructure
// flakiness when running many sequential Docker-based integration tests.
// excludedFromNixChecks lists tests the hermetic nix-VM checks cannot run, with
// the reason. They still run via cmd/hi, where the missing resource exists.
var excludedFromNixChecks = map[string]string{
// Verifies connectivity through Tailscale's *public* DERP relays
// (HEADSCALE_DERP_URLS=controlplane.tailscale.com). A hermetic offline VM
// has no internet and the nix sandbox forbids network.
"TestPingAllByIPPublicDERP": "requires public internet DERP",
// Also routes DERP through Tailscale's public relays: tailscale-rs cannot
// validate the embedded DERP's self-signed cert (WithPublicDERP in the
// test), so it needs the same internet access PublicDERP does.
"TestTailscaleRustAxum": "requires public internet DERP (tailscale-rs)",
// These drive `docker network disconnect`/`connect` to simulate cable pulls.
// In the VM's nested docker, libnetwork intermittently fails the reconnect
// ("failed to set gateway: network is unreachable" — a stale IPAM/endpoint
// reservation), which does not happen against a host docker daemon. They run
// reliably via cmd/hi in the legacy pipeline.
"TestHASubnetRouterFailoverDockerDisconnect": "docker reconnect unreliable in nested VM docker",
"TestHASubnetRouterFailoverBothOfflineCablePull": "docker reconnect unreliable in nested VM docker",
// Ping-driven HA failover: after a router recovers it asserts router 2 stays
// primary and traffic still traceroutes through it. In the VM's nested docker
// that data-plane state never converges (a 4x ScaledTimeout budget did not
// help — behaviour, not timing); it holds against a host docker daemon.
"TestHASubnetRouterPingFailover": "ping-failover data plane doesn't converge in nested VM docker",
}
// testsToSplit defines tests whose subtests are run as separate checks. Key is
// the test function, value is a list of subtest prefixes; each prefix becomes
// its own check as "TestName/prefix.*". TestAutoApproveMultiNetwork fans out
// over approver × policy-mode × advertise (24 subtests); running all of them in
// one nested-docker VM check overruns the timeout, so split by approver into
// 6 checks of 4 — fewer sequential docker scenarios per check.
var testsToSplit = map[string][]string{
"TestAutoApproveMultiNetwork": {
"authkey-tag",
@ -39,26 +58,32 @@ var testsToSplit = map[string][]string{
},
}
// expandTests takes a list of test names and expands any that need splitting
// into multiple subtest patterns.
// expandTests replaces any splittable test with one entry per subtest prefix.
// The ".*" lets the check's "^...$"-wrapped -test.run match the full subtest
// names (e.g. authkey-tag-advertiseduringup-false-pol-database).
func expandTests(tests []string) []string {
var expanded []string
expanded := make([]string, 0, len(tests))
for _, test := range tests {
if prefixes, ok := testsToSplit[test]; ok {
// This test should be split into multiple jobs.
// We append ".*" to each prefix because the CI runner wraps patterns
// with ^...$ anchors. Without ".*", a pattern like "authkey$" wouldn't
// match "authkey-tag-advertiseduringup-false-pol-database".
for _, prefix := range prefixes {
expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix))
}
} else {
prefixes, ok := testsToSplit[test]
if !ok {
expanded = append(expanded, test)
continue
}
for _, prefix := range prefixes {
expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix))
}
}
return expanded
}
// This generator enumerates the integration test functions and writes them as
// nix lists (integration/tests.nix and integration/postgres-tests.nix). The
// flake maps each entry to a NixOS-VM check (sqlite for all, plus a postgres
// variant for the subset) — see nix/tests/integration.nix. One check per test
// function; the GitHub workflow derives its matrix from these checks.
func findTests() []string {
rgBin, err := exec.LookPath("rg")
if err != nil {
@ -89,43 +114,41 @@ func findTests() []string {
return tests
}
func updateYAML(tests []string, jobName string, testPath string) {
testsForYq := fmt.Sprintf("[%s]", strings.Join(tests, ", "))
// writeNixList writes the test function names as a nix list, consumed by
// flake.nix to generate one integration check per test.
func writeNixList(tests []string, nixPath string) {
var b strings.Builder
b.WriteString("# Generated by gh-action-integration-generator.go; do not edit.\n")
b.WriteString("[\n")
for _, test := range tests {
fmt.Fprintf(&b, " %q\n", test)
}
b.WriteString("]\n")
yqCommand := fmt.Sprintf(
"yq eval '.jobs.%s.strategy.matrix.test = %s' %s -i",
jobName,
testsForYq,
testPath,
)
cmd := exec.Command("bash", "-c", yqCommand)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
log.Printf("stdout: %s", stdout.String())
log.Printf("stderr: %s", stderr.String())
log.Fatalf("failed to run yq command: %s", err)
if err := os.WriteFile(nixPath, []byte(b.String()), 0o644); err != nil {
log.Fatalf("failed to write %s: %s", nixPath, err)
}
fmt.Printf("YAML file (%s) job %s updated successfully\n", testPath, jobName)
fmt.Printf("Nix list (%s) updated successfully\n", nixPath)
}
func main() {
tests := findTests()
// Expand tests that should be split into multiple jobs
expandedTests := expandTests(tests)
tests = slices.DeleteFunc(tests, func(name string) bool {
if reason, ok := excludedFromNixChecks[name]; ok {
log.Printf("excluding %s from nix checks: %s", name, reason)
return true
}
return false
})
quotedTests := make([]string, len(expandedTests))
for i, test := range expandedTests {
quotedTests[i] = fmt.Sprintf("\"%s\"", test)
}
// Split heavy tests into per-subtest checks (after exclusion, so an excluded
// test is never expanded).
tests = expandTests(tests)
// Define selected tests for PostgreSQL
// Selected tests also run against PostgreSQL to verify database
// compatibility (a postgres-variant check per name).
postgresTestNames := []string{
"TestACLAllowUserDst",
"TestPingAllByIP",
@ -134,12 +157,30 @@ func main() {
"TestSubnetRouterMultiNetwork",
}
quotedPostgresTests := make([]string, len(postgresTestNames))
for i, test := range postgresTestNames {
quotedPostgresTests[i] = fmt.Sprintf("\"%s\"", test)
// Tests excluded from the hermetic nix checks still need to run somewhere,
// so emit them for the legacy cmd/hi GitHub-Actions matrix (real internet).
excluded := make([]string, 0, len(excludedFromNixChecks))
for name := range excludedFromNixChecks {
excluded = append(excluded, name)
}
slices.Sort(excluded)
writeNixList(tests, "../../integration/tests.nix")
writeNixList(postgresTestNames, "../../integration/postgres-tests.nix")
writeJSONList(excluded, "../../integration/excluded-tests.json")
}
// writeJSONList writes the names as a JSON array, consumed as a GitHub-Actions
// matrix by the legacy cmd/hi workflow.
func writeJSONList(names []string, path string) {
data, err := json.MarshalIndent(names, "", " ")
if err != nil {
log.Fatalf("marshalling %s: %s", path, err)
}
// Update both SQLite and PostgreSQL job matrices
updateYAML(quotedTests, "sqlite", "./test-integration.yaml")
updateYAML(quotedPostgresTests, "postgres", "./test-integration.yaml")
if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil {
log.Fatalf("failed to write %s: %s", path, err)
}
fmt.Printf("JSON list (%s) updated successfully\n", path)
}

146
.github/workflows/integration-test-template.yml vendored
View file

@ -1,146 +0,0 @@
name: Integration Test Template
on:
workflow_call:
inputs:
test:
required: true
type: string
postgres_flag:
required: false
type: string
default: ""
database_name:
required: true
type: string
jobs:
test:
runs-on: ubuntu-24.04-arm
env:
# Github does not allow us to access secrets in pull requests,
# so this env var is used to check if we have the secret or not.
# If we have the secrets, meaning we are running on push in a fork,
# there might be secrets available for more debugging.
# If TS_OAUTH_CLIENT_ID and TS_OAUTH_SECRET is set, then the job
# will join a debug tailscale network, set up SSH and a tmux session.
# The SSH will be configured to use the SSH key of the Github user
# that triggered the build.
HAS_TAILSCALE_SECRET: ${{ secrets.TS_OAUTH_CLIENT_ID }}
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
fetch-depth: 2
- name: Tailscale
if: ${{ env.HAS_TAILSCALE_SECRET }}
uses: tailscale/github-action@a392da0a182bba0e9613b6243ebd69529b1878aa # v4.1.0
with:
oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
tags: tag:gh
- name: Setup SSH server for Actor
if: ${{ env.HAS_TAILSCALE_SECRET }}
uses: alexellis/setup-sshd-actor@master
- name: Download headscale image
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: headscale-image
path: /tmp/artifacts
- name: Download tailscale HEAD image
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: tailscale-head-image
path: /tmp/artifacts
- name: Download tailscale released images
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: tailscale-released-images
path: /tmp/artifacts
- name: Download hi binary
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: hi-binary
path: /tmp/artifacts
- name: Download Go cache
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: go-cache
path: /tmp/artifacts
- name: Download postgres image
if: ${{ inputs.postgres_flag == '--postgres=1' }}
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: postgres-image
path: /tmp/artifacts
- name: Force overlay2 storage driver
run: |
sudo mkdir -p /etc/docker
echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
docker version
- name: Load br_netfilter for in-cluster service routing
if: inputs.test == 'TestK8sOperator'
# TestK8sOperator runs k3s in a container; without br_netfilter on the
# host, bridged pod-to-pod traffic skips kube-proxy's ClusterIP DNAT and
# in-cluster DNS (kube-dns) is unreachable. The module cannot be loaded
# from inside the unprivileged-module rancher/k3s image, so load it here.
run: sudo modprobe br_netfilter
- name: Login to Docker Hub
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
if: env.DOCKERHUB_USERNAME != ''
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ env.DOCKERHUB_TOKEN }}
- name: Load Docker images, Go cache, and prepare binary
run: |
gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load
gunzip -c /tmp/artifacts/tailscale-head-image.tar.gz | docker load
gunzip -c /tmp/artifacts/tailscale-released-images.tar.gz | docker load
if [ -f /tmp/artifacts/postgres-image.tar.gz ]; then
gunzip -c /tmp/artifacts/postgres-image.tar.gz | docker load
fi
chmod +x /tmp/artifacts/hi
docker images
# Extract Go cache to host directories for bind mounting
mkdir -p /tmp/go-cache
tar -xzf /tmp/artifacts/go-cache.tar.gz -C /tmp/go-cache
ls -la /tmp/go-cache/ /tmp/go-cache/.cache/
- name: Run Integration Test
env:
HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:${{ github.sha }}
HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:${{ github.sha }}
HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }}
HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go
HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build
# Mirror the docker/login-action secrets into env so the
# dockertestutil.Credentials resolver picks them up directly
# (otherwise it falls back to parsing ~/.docker/config.json,
# which works but is one step further from the source).
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \
--timeout=120m \
${{ inputs.postgres_flag }}
# Sanitize test name for artifact upload (replace invalid characters: " : < > | * ? \ / with -)
- name: Sanitize test name for artifacts
if: always()
id: sanitize
run: echo "name=${TEST_NAME//[\":<>|*?\\\/]/-}" >> $GITHUB_OUTPUT
env:
TEST_NAME: ${{ inputs.test }}
- uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
if: always()
with:
name: ${{ inputs.database_name }}-${{ steps.sanitize.outputs.name }}-logs
path: "control_logs/*/*.log"
- uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
if: always()
with:
name: ${{ inputs.database_name }}-${{ steps.sanitize.outputs.name }}-artifacts
path: control_logs/
- name: Setup a blocking tmux session
if: ${{ env.HAS_TAILSCALE_SECRET }}
uses: alexellis/block-with-tmux-action@master

56
.github/workflows/nix-checks.yml vendored
View file

@ -1,56 +0,0 @@
name: Nix Flake Checks
on:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
cancel-in-progress: true
# Each job only runs `nix build .#checks.<system>.<name>`; the check logic lives
# in flake.nix via the flake-checks library. The fileset-filtered checks hit the
# hestia cache when their inputs are unchanged, so no changed-files gating.
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
- name: build
run: nix build -L .#checks.x86_64-linux.build
gotest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
- name: gotest
run: nix build -L .#checks.x86_64-linux.gotest
golangci-lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
- name: golangci-lint
run: nix build -L .#checks.x86_64-linux.golangci-lint
formatting:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
- name: formatting
run: nix build -L .#checks.x86_64-linux.formatting

51
.github/workflows/nix-module-test.yml vendored
View file

@ -1,51 +0,0 @@
name: NixOS Module Tests
on:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
nix-module-check:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
with:
filters: |
nix:
- 'nix/**'
- 'flake.nix'
- 'flake.lock'
go:
- 'go.*'
- '**/*.go'
- 'cmd/**'
- 'hscontrol/**'
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true'
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true'
- name: Run NixOS module tests
if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true'
run: |
echo "Running NixOS module integration test..."
nix build .#checks.x86_64-linux.headscale -L --fallback

75
.github/workflows/test-integration-legacy.yaml vendored Normal file
View file

@ -0,0 +1,75 @@
name: integration-legacy
on:
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
# A small number of integration tests cannot run as hermetic NixOS-VM checks
# because they need Tailscale's public DERP relays (real internet) — see
# excludedFromNixChecks in gh-action-integration-generator.go. They run here the
# legacy way, via cmd/hi against Dockerfile-built images, which also keeps the
# cmd/hi path and the integration Dockerfiles exercised in CI. The matrix is the
# generated integration/excluded-tests.json so this list has a single source.
jobs:
list:
runs-on: ubuntu-latest
outputs:
tests: ${{ steps.list.outputs.tests }}
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- id: list
run: echo "tests=$(jq -c . integration/excluded-tests.json)" >> "$GITHUB_OUTPUT"
test:
needs: list
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test: ${{ fromJSON(needs.list.outputs.tests) }}
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
# Docker 29's default containerd snapshotter regresses these images; the
# classic overlay2 graph driver is the known-good path (mirrors the VM checks).
- name: Force overlay2 storage driver
run: |
sudo mkdir -p /etc/docker
echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Login to Docker Hub
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
if: env.DOCKERHUB_USERNAME != ''
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ env.DOCKERHUB_TOKEN }}
# cmd/hi requires prebuilt head images in CI (util.IsCI); released versions
# pull from the registry and tailscale-rs is built inline by tsric.
# Dockerfile.integration-ci COPYs a prebuilt headscale binary from the
# build context, so build it first (tailscale-HEAD builds from source).
- name: Build headscale + tailscale-HEAD images
run: |
nix develop --command bash -c 'CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale'
docker build --file Dockerfile.integration-ci --tag headscale:ci .
docker build --file Dockerfile.tailscale-HEAD --tag tailscale-head:ci .
- name: Run ${{ matrix.test }}
env:
HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:ci
HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:ci
run: nix develop --command go run ./cmd/hi run "^${{ matrix.test }}$" --timeout=900s

406
.github/workflows/test-integration.yaml vendored
View file

@ -1,406 +0,0 @@
name: integration
# To debug locally on a branch, and when needing secrets
# change this to include `push` so the build is ran on
# the main repository.
on: [pull_request]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
defaults:
run:
shell: nix develop --fallback --command bash -e {0}
jobs:
# build: Builds binaries and Docker images once, uploads as artifacts for reuse.
# build-postgres: Pulls postgres image separately to avoid Docker Hub rate limits.
# build-tailscale-released: Pre-pulls released Tailscale images from ghcr.io
# so fork PRs (no DOCKERHUB_USERNAME secret) don't hit Docker Hub rate
# limits at test time.
# sqlite: Runs all integration tests with SQLite backend.
# postgres: Runs a subset of tests with PostgreSQL to verify database compatibility.
build:
runs-on: ubuntu-24.04-arm
outputs:
files-changed: ${{ steps.changed-files.outputs.files }}
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
with:
filters: |
files:
- '*.nix'
- 'go.*'
- '**/*.go'
- 'integration/**'
- 'config-example.yaml'
- '.github/workflows/test-integration.yaml'
- '.github/workflows/integration-test-template.yml'
- 'Dockerfile.*'
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
if: steps.changed-files.outputs.files == 'true'
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
if: steps.changed-files.outputs.files == 'true'
- name: Build binaries and warm Go cache
if: steps.changed-files.outputs.files == 'true'
run: |
# Build all Go binaries in one nix shell to maximize cache reuse
go build -o hi ./cmd/hi
CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale
# Build integration test binary to warm the cache with all dependencies
go test -c ./integration -o /dev/null 2>/dev/null || true
- name: Upload hi binary
if: steps.changed-files.outputs.files == 'true'
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: hi-binary
path: hi
retention-days: 10
- name: Package Go cache
if: steps.changed-files.outputs.files == 'true'
run: |
# Package Go module cache and build cache
tar -czf go-cache.tar.gz -C ~ go .cache/go-build
- name: Upload Go cache
if: steps.changed-files.outputs.files == 'true'
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: go-cache
path: go-cache.tar.gz
retention-days: 10
- name: Force overlay2 storage driver
if: steps.changed-files.outputs.files == 'true'
run: |
# Docker 29 runner images default to overlayfs, which breaks
# docker build via Go SDK libraries and docker save/load
# tarball formats. overlay2 is the long-standing default.
# https://github.com/actions/runner-images/issues/13474
sudo mkdir -p /etc/docker
echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
docker version
- name: Login to Docker Hub
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
if: env.DOCKERHUB_USERNAME != ''
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ env.DOCKERHUB_TOKEN }}
- name: Build headscale image
if: steps.changed-files.outputs.files == 'true'
run: |
docker build \
--file Dockerfile.integration-ci \
--tag headscale:${{ github.sha }} \
.
docker save headscale:${{ github.sha }} | gzip > headscale-image.tar.gz
- name: Build tailscale HEAD image
if: steps.changed-files.outputs.files == 'true'
run: |
docker build \
--file Dockerfile.tailscale-HEAD \
--tag tailscale-head:${{ github.sha }} \
.
docker save tailscale-head:${{ github.sha }} | gzip > tailscale-head-image.tar.gz
- name: Upload headscale image
if: steps.changed-files.outputs.files == 'true'
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: headscale-image
path: headscale-image.tar.gz
retention-days: 10
- name: Upload tailscale HEAD image
if: steps.changed-files.outputs.files == 'true'
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: tailscale-head-image
path: tailscale-head-image.tar.gz
retention-days: 10
build-postgres:
runs-on: ubuntu-24.04-arm
needs: build
if: needs.build.outputs.files-changed == 'true'
steps:
- name: Force overlay2 storage driver
shell: bash
run: |
sudo mkdir -p /etc/docker
echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
docker version
- name: Login to Docker Hub
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
if: env.DOCKERHUB_USERNAME != ''
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ env.DOCKERHUB_TOKEN }}
- name: Pull and save postgres image
shell: bash
run: |
docker pull postgres:latest
docker tag postgres:latest postgres:${{ github.sha }}
docker save postgres:${{ github.sha }} | gzip > postgres-image.tar.gz
- name: Upload postgres image
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: postgres-image
path: postgres-image.tar.gz
retention-days: 10
build-tailscale-released:
runs-on: ubuntu-24.04-arm
needs: build
if: needs.build.outputs.files-changed == 'true'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
- uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main
- name: Force overlay2 storage driver
run: |
sudo mkdir -p /etc/docker
echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
docker version
- name: Login to Docker Hub
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
if: env.DOCKERHUB_USERNAME != ''
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ env.DOCKERHUB_TOKEN }}
- name: List Tailscale versions to pre-pull
id: versions
run: |
versions=$(go run ./cmd/hi list-versions --set=must --exclude=head)
echo "versions=${versions}" >> "$GITHUB_OUTPUT"
echo "Pre-pulling: ${versions}"
- name: Pull Tailscale images
run: |
# Releases come from ghcr.io (anonymous, unmetered). The
# "unstable" floating tag on ghcr.io has been stale since 2022,
# so it still needs to come from Docker Hub. xargs -P 0 fans
# out one process per tag and returns non-zero if any pull
# fails.
refs=""
for v in ${{ steps.versions.outputs.versions }}; do
if [ "${v}" = "unstable" ]; then
refs="${refs} tailscale/tailscale:${v}"
else
refs="${refs} ghcr.io/tailscale/tailscale:${v}"
fi
done
echo "${refs}" | tr ' ' '\n' | grep -v '^$' \
| xargs -P 0 -I{} docker pull "{}"
echo "REFS=${refs}" >> "$GITHUB_ENV"
- name: Save Tailscale images to tarball
run: |
# Single docker save with all refs: one consistent snapshot, no
# parallel-daemon race.
docker save ${REFS} | gzip > tailscale-released-images.tar.gz
ls -lh tailscale-released-images.tar.gz
- name: Upload Tailscale released images
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: tailscale-released-images
path: tailscale-released-images.tar.gz
retention-days: 10
sqlite:
needs: [build, build-tailscale-released]
if: needs.build.outputs.files-changed == 'true'
strategy:
fail-fast: false
matrix:
test:
- TestACLHostsInNetMapTable
- TestACLAllowUser80Dst
- TestACLDenyAllPort80
- TestACLAllowUserDst
- TestACLAllowStarDst
- TestACLNamedHostsCanReachBySubnet
- TestACLNamedHostsCanReach
- TestACLDevice1CanAccessDevice2
- TestPolicyUpdateWhileRunningWithCLIInDatabase
- TestACLAutogroupMember
- TestACLAutogroupTagged
- TestACLAutogroupSelf
- TestACLPolicyPropagationOverTime
- TestACLTagPropagation
- TestACLTagPropagationPortSpecific
- TestACLGroupWithUnknownUser
- TestACLGroupAfterUserDeletion
- TestACLGroupDeletionExactReproduction
- TestACLDynamicUnknownUserAddition
- TestACLDynamicUnknownUserRemoval
- TestAPIAuthenticationBypass
- TestAPIAuthenticationBypassCurl
- TestRemoteCLIAuthenticationBypass
- TestCLIWithConfigAuthenticationBypass
- TestAuthKeyLogoutAndReloginSameUser
- TestAuthKeyLogoutAndReloginNewUser
- TestAuthKeyLogoutAndReloginSameUserExpiredKey
- TestAuthKeyDeleteKey
- TestAuthKeyLogoutAndReloginRoutesPreserved
- TestOIDCAuthenticationPingAll
- TestOIDCExpireNodesBasedOnTokenExpiry
- TestOIDC024UserCreation
- TestOIDCAuthenticationWithPKCE
- TestOIDCReloginSameNodeNewUser
- TestOIDCFollowUpUrl
- TestOIDCMultipleOpenedLoginUrls
- TestOIDCReloginSameNodeSameUser
- TestOIDCExpiryAfterRestart
- TestOIDCACLPolicyOnJoin
- TestOIDCReloginSameUserRoutesPreserved
- TestAuthWebFlowAuthenticationPingAll
- TestAuthWebFlowLogoutAndReloginSameUser
- TestAuthWebFlowLogoutAndReloginNewUser
- TestApiKeyCommand
- TestApiKeyCommandValidation
- TestAuthCommandValidation
- TestNodeCommand
- TestNodeExpireCommand
- TestNodeRenameCommand
- TestPreAuthKeyCorrectUserLoggedInCommand
- TestTaggedNodesCLIOutput
- TestNodeExpireFlagsCommand
- TestNodeCommandValidation
- TestNodeTagCommand
- TestNodeRouteCommands
- TestNodeBackfillIPsCommand
- TestOAuthClientCommand
- TestOAuthClientCommandValidation
- TestPolicyCheckCommand
- TestSSHTestsRejectFailingPolicy
- TestPolicyCommand
- TestPolicyBrokenConfigCommand
- TestPreAuthKeyCommand
- TestPreAuthKeyCommandWithoutExpiry
- TestPreAuthKeyCommandReusableEphemeral
- TestPreAuthKeyDeleteCommand
- TestPreAuthKeyCommandValidation
- TestServerInfoCommands
- TestUserCommand
- TestUserCreateCommand
- TestUserCommandValidation
- TestDERPVerifyEndpoint
- TestResolveMagicDNS
- TestResolveMagicDNSExtraRecordsPath
- TestDERPServerScenario
- TestDERPServerWebsocketScenario
- TestPingAllByIP
- TestPingAllByIPPublicDERP
- TestEphemeral
- TestEphemeralInAlternateTimezone
- TestEphemeral2006DeletedTooQuickly
- TestPingAllByHostname
- TestTaildrop
- TestUpdateHostnameFromClient
- TestExpireNode
- TestSetNodeExpiryInFuture
- TestDisableNodeExpiry
- TestNodeOnlineStatus
- TestPingAllByIPManyUpDown
- Test2118DeletingOnlineNodePanics
- TestGrantCapRelay
- TestGrantCapDrive
- TestK8sOperator
- TestEnablingRoutes
- TestHASubnetRouterFailover
- TestSubnetRouteACL
- TestEnablingExitRoutes
- TestExitRoutesWithAutogroupInternetACL
- TestSubnetRouterMultiNetwork
- TestSubnetRouterMultiNetworkExitNode
- TestAutoApproveMultiNetwork/authkey-tag.*
- TestAutoApproveMultiNetwork/authkey-user.*
- TestAutoApproveMultiNetwork/authkey-group.*
- TestAutoApproveMultiNetwork/webauth-tag.*
- TestAutoApproveMultiNetwork/webauth-user.*
- TestAutoApproveMultiNetwork/webauth-group.*
- TestSubnetRouteACLFiltering
- TestGrantViaSubnetSteering
- TestHASubnetRouterPingFailover
- TestHASubnetRouterFailoverBothOffline
- TestHASubnetRouterFailoverBothOfflineCablePull
- TestHASubnetRouterFailoverDockerDisconnect
- TestHeadscale
- TestTailscaleNodesJoiningHeadcale
- TestSSHOneUserToAll
- TestSSHMultipleUsersAllToAll
- TestSSHNoSSHConfigured
- TestSSHIsBlockedInACL
- TestSSHUserOnlyIsolation
- TestSSHAutogroupSelf
- TestSSHOneUserToOneCheckModeCLI
- TestSSHOneUserToOneCheckModeOIDC
- TestSSHCheckModeUnapprovedTimeout
- TestSSHCheckModeCheckPeriodCLI
- TestSSHCheckModeAutoApprove
- TestSSHCheckModeSessionLossReDelegates
- TestSSHCheckModeNegativeCLI
- TestSSHLocalpart
- TestTagsAuthKeyWithTagRequestDifferentTag
- TestTagsAuthKeyWithTagNoAdvertiseFlag
- TestTagsAuthKeyWithTagCannotAddViaCLI
- TestTagsAuthKeyWithTagCannotChangeViaCLI
- TestTagsAuthKeyWithTagAdminOverrideReauthPreserves
- TestTagsAuthKeyWithTagCLICannotModifyAdminTags
- TestTagsAuthKeyWithoutTagCannotRequestTags
- TestTagsAuthKeyWithoutTagRegisterNoTags
- TestTagsAuthKeyWithoutTagCannotAddViaCLI
- TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset
- TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise
- TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag
- TestTagsUserLoginOwnedTagAtRegistration
- TestTagsUserLoginNonExistentTagAtRegistration
- TestTagsUserLoginUnownedTagAtRegistration
- TestTagsUserLoginAddTagViaCLIReauth
- TestTagsUserLoginRemoveTagViaCLIReauth
- TestTagsUserLoginCLINoOpAfterAdminAssignment
- TestTagsUserLoginCLICannotRemoveAdminTags
- TestTagsAuthKeyWithTagRequestNonExistentTag
- TestTagsAuthKeyWithTagRequestUnownedTag
- TestTagsAuthKeyWithoutTagRequestNonExistentTag
- TestTagsAuthKeyWithoutTagRequestUnownedTag
- TestTagsAdminAPICannotSetNonExistentTag
- TestTagsAdminAPICanSetUnownedTag
- TestTagsAdminAPICannotRemoveAllTags
- TestTagsIssue2978ReproTagReplacement
- TestTagsAdminAPICannotSetInvalidFormat
- TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags
- TestTagsAuthKeyWithoutUserInheritsTags
- TestTagsAuthKeyWithoutUserRejectsAdvertisedTags
- TestTagsAuthKeyConvertToUserViaCLIRegister
- TestTailscaleRustAxum
uses: ./.github/workflows/integration-test-template.yml
secrets: inherit
with:
test: ${{ matrix.test }}
postgres_flag: "--postgres=0"
database_name: "sqlite"
postgres:
needs: [build, build-postgres, build-tailscale-released]
if: needs.build.outputs.files-changed == 'true'
strategy:
fail-fast: false
matrix:
test:
- TestACLAllowUserDst
- TestPingAllByIP
- TestEphemeral2006DeletedTooQuickly
- TestPingAllByIPManyUpDown
- TestSubnetRouterMultiNetwork
uses: ./.github/workflows/integration-test-template.yml
secrets: inherit
with:
test: ${{ matrix.test }}
postgres_flag: "--postgres=1"
database_name: "postgres"

4
.prettierignore
View file

@ -1,2 +1,6 @@
.github/workflows/test-integration-v2*
docs/
# Generated by gh-action-integration-generator.go (its own canonical format).
integration/excluded-tests.json
# Generated by update-integration-images (jq output).
integration/tailscale-versions.json

4
cmd/hi/listversions.go
View file

@ -30,9 +30,7 @@ var listVersionsConfig ListVersionsConfig
// tags, releases get a "v" prefix so each entry can be appended to
// "ghcr.io/tailscale/tailscale:" directly.
func listVersions(env *command.Env) error {
release := capver.TailscaleLatestMajorMinor(capver.SupportedMajorMinorVersions, true)
all := append([]string{"head", "unstable"}, release...)
must := append(append([]string{}, all[0:4]...), all[len(all)-2:]...)
all, must := capver.IntegrationVersions()
var versions []string

70
flake.lock generated
View file

@ -22,6 +22,22 @@
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1767039857,
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
@ -74,11 +90,28 @@
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1772736753,
"narHash": "sha256-au/m3+EuBLoSzWUCb64a/MZq6QUtOV8oC0D9tY2scPQ=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "917fec990948658ef1ccd07cef2a1ef060786846",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-checks": "flake-checks",
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs"
"nixpkgs": "nixpkgs",
"tailscale-head": "tailscale-head"
}
},
"systems": {
@ -111,6 +144,41 @@
"type": "github"
}
},
"systems_3": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"tailscale-head": {
"inputs": {
"flake-compat": "flake-compat",
"nixpkgs": "nixpkgs_2",
"systems": "systems_3"
},
"locked": {
"lastModified": 1782250635,
"narHash": "sha256-y4/EAApBPekjWz2cP/9RexTBkiSAXsdOHWnhm/no/4I=",
"owner": "tailscale",
"repo": "tailscale",
"rev": "d4f2917c1bfd27529ec7997d08a8f1aa9ea903f2",
"type": "github"
},
"original": {
"owner": "tailscale",
"repo": "tailscale",
"type": "github"
}
},
"treefmt-nix": {
"inputs": {
"nixpkgs": [

138
flake.nix
View file

@ -9,10 +9,18 @@
# once it ships go_1_26 >= 1.26.4.
nixpkgs.url = "github:NixOS/nixpkgs/staging-next-26.05";
flake-utils.url = "github:numtide/flake-utils";
# Reusable Go flake checks (build/test/lint/format); CI runs them via
# `nix build .#checks.<system>.<name>` instead of bespoke per-tool steps.
flake-checks.url = "github:kradalby/flake-checks";
flake-checks.inputs.nixpkgs.follows = "nixpkgs";
# Tailscale HEAD, built from source for the integration test client image.
# Bump with `nix flake update tailscale-head` to track the latest main.
# It is a flake; we still build our own derivation (we need derper +
# containerboot, which its default package omits) but read its committed
# vendorHash from flakehashes.json so the bump carries the hash.
tailscale-head.url = "github:tailscale/tailscale";
};
outputs =
@ -20,6 +28,7 @@
, nixpkgs
, flake-utils
, flake-checks
, tailscale-head
, ...
}:
let
@ -39,8 +48,27 @@
# Go 1.26 builder; resolves to Go 1.26.4 from the pinned nixpkgs.
buildGo = pkgs.buildGo126Module;
vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri;
# Go source with the non-Go scaffolding filtered out, so editing the
# nix integration harness, CI workflows, docs or packaging does not
# rebuild the (slow) integration test binary or container images.
# Only changes to actual Go source invalidate them.
goSrc = pkgs.lib.fileset.toSource {
root = ./.;
fileset = pkgs.lib.fileset.difference ./. (pkgs.lib.fileset.unions [
./nix
./.github
./docs
./packaging
./flake.nix
./flake.lock
./flakehashes.json
]);
};
in
{
headscale-go-src = goSrc;
headscale = buildGo {
pname = "headscale";
version = headscaleVersion;
@ -71,6 +99,34 @@
subPackages = [ "cmd/hi" ];
};
# The integration suite compiled into a single test binary, run later
# inside a per-test NixOS VM check (see nix/tests/integration.nix).
# Compiled once and shared by every integration check.
integration-test-bin = buildGo {
pname = "headscale-integration-test";
# Fixed (not the commit rev) so the binary is content-addressed:
# identical Go source → identical store path → shared-cache hit
# across commits. The version string is irrelevant for a test binary.
version = "integration";
src = goSrc;
inherit vendorHash;
doCheck = false;
buildPhase = ''
runHook preBuild
go test -c -o integration.test ./integration
runHook postBuild
'';
installPhase = ''
runHook preInstall
install -Dm755 integration.test $out/bin/integration.test
runHook postInstall
'';
};
# Build golangci-lint with stock Go 1.26 (upstream uses hardcoded Go
# version); it does not build against the pinned 1.26.4.
golangci-lint = buildGo rec {
@ -200,7 +256,12 @@
vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri;
goPkg = pkgs.go_1_26;
# //go:embed targets and test-read files outside the default whitelist.
embedDirs = [ ./hscontrol/assets ./hscontrol/db/schema.sql ./config-example.yaml ];
embedDirs = [
./hscontrol/assets
./hscontrol/db/schema.sql
./config-example.yaml
./integration/tailscale-versions.json
];
extraSrc = [
./hscontrol/testdata
./hscontrol/types/testdata
@ -239,6 +300,45 @@
fmtExclude = [ ./gen ./docs ];
});
};
# Nix-built container images for the integration tests, plus the
# per-test VM check factory. Linux-only (dockerTools + nixosTest).
integrationImages = import ./nix/images.nix {
inherit pkgs;
buildGoModule = pkgs.buildGo126Module;
tailscaleSrc = tailscale-head;
};
mkIntegrationCheck = { name, testFilter ? name, postgres ? false }:
pkgs.testers.nixosTest (import ./nix/tests/integration.nix {
inherit name testFilter pkgs postgres;
inherit (integrationImages)
headscaleImage tailscaleImage postgresImage tailscaleVersionImages;
testBin = pkgs.integration-test-bin;
# goSrc (not the full tree) so a check's result stays cached when
# only nix/CI/docs change — only real Go/integration changes (which
# also rebuild testBin) invalidate it. Maximises shared-cache reuse.
src = pkgs.headscale-go-src;
});
# One sqlite check per test (full matrix), plus a postgres variant for
# the postgres subset — exactly the sqlite+postgres jobs the generator
# produces for the GitHub workflow.
integrationChecks =
let
# Split-test entries (TestAutoApproveMultiNetwork/authkey-tag.*) carry
# / and * in their -test.run filter; sanitise those into a valid check
# name while keeping the raw filter for -test.run.
sanitize = builtins.replaceStrings [ "/" "." "*" ] [ "-" "" "" ];
mkCheck = postgres: filter:
pkgs.lib.nameValuePair
"integration-${sanitize filter}${pkgs.lib.optionalString postgres "-pg"}"
(mkIntegrationCheck { name = sanitize filter; testFilter = filter; inherit postgres; });
in
pkgs.lib.listToAttrs (
map (mkCheck false) (import ./integration/tests.nix)
++ map (mkCheck true) (import ./integration/postgres-tests.nix)
);
in
{
# `nix develop`
@ -259,6 +359,14 @@
cat go.mod | ${pkgs.ripgrep}/bin/rg "\t" | ${pkgs.ripgrep}/bin/rg -v indirect | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.findutils}/bin/xargs go get -u
go mod tidy
'')
# Regenerate integration/tailscale-versions.json after a capver bump
# so the offline checks pin the tailscale images the suite requests.
# It is a //go:generate step in the integration package (runs with
# capver under `make generate`); this just scopes it to the pins.
(pkgs.writeShellScriptBin "update-integration-images" ''
exec go generate ./integration/...
'')
];
shellHook = ''
@ -272,6 +380,28 @@
inherit headscale;
inherit headscale-docker;
default = headscale;
}
// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux {
headscale-integration-image = integrationImages.headscaleImage;
tailscale-integration-image = integrationImages.tailscaleImage;
postgres-integration-image = integrationImages.postgresImage;
inherit (pkgs) integration-test-bin;
# All shared inputs every integration check pulls in: the test binary
# and every image. CI builds this once (a "prime" job) so it lands in
# the shared nix cache before the per-test matrix fans out — the
# matrix jobs then substitute it instead of each rebuilding.
integration-deps = pkgs.linkFarm "integration-deps" (
[
{ name = "test-bin"; path = pkgs.integration-test-bin; }
{ name = "headscale"; path = integrationImages.headscaleImage; }
{ name = "tailscale-head"; path = integrationImages.tailscaleImage; }
{ name = "postgres"; path = integrationImages.postgresImage; }
]
++ pkgs.lib.imap0
(i: img: { name = "ts-version-${toString i}"; path = img; })
integrationImages.tailscaleVersionImages
);
};
# `nix run`
@ -287,6 +417,10 @@
}
# The Go build/test checks are gated to Linux: parts of the tree are
# Linux-specific and the pure unit subset is validated by CI.
// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks;
// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks
# Integration checks only on x86_64-linux: nixosTest needs KVM and the
# pinned version images are amd64 (nix/tailscale-versions.nix). aarch64
# would need arch-specific pins.
// pkgs.lib.optionalAttrs (system == "x86_64-linux") integrationChecks;
});
}

14
hscontrol/capver/capver.go
View file

@ -84,3 +84,17 @@ func TailscaleLatestMajorMinor(n int, stripV bool) []string {
return majorSl[len(majorSl)-n:]
}
// IntegrationVersions returns the bare Tailscale version tags the integration
// suite pins and tests — the full set and the "must" subset (head, unstable,
// the two newest and two oldest tracked releases, for boundary coverage).
// Releases are bare (1.80); callers that build image tags prepend "v". This is
// the one source the suite's version list, the drift guard (`hi list-versions`),
// and the tailscale-versions.json generator all derive from.
func IntegrationVersions() ([]string, []string) {
release := TailscaleLatestMajorMinor(SupportedMajorMinorVersions, true)
all := append([]string{"head", "unstable"}, release...)
must := append(append([]string{}, all[0:4]...), all[len(all)-2:]...)
return all, must
}

15
integration/README.md
View file

@ -21,16 +21,19 @@ go run ./cmd/hi doctor
go run ./cmd/hi run "TestPingAllByIP"
```
Alternatively, [`act`](https://github.com/nektos/act) runs the GitHub
Actions workflow locally:
In CI each test also runs as its own hermetic NixOS-VM flake check. Run
one locally exactly the way CI does:
```bash
act pull_request -W .github/workflows/test-integration.yaml
nix build .#checks.x86_64-linux.integration-TestPingAllByIP
```
Each test runs as a separate workflow on GitHub Actions. To add a new
test, run `go generate` inside `../cmd/gh-action-integration-generator/`
and commit the generated workflow file.
The check matrix is generated: after adding or renaming a test, run `go
generate` in `.github/workflows/` to refresh `integration/tests.nix`,
`integration/postgres-tests.nix`, and `integration/excluded-tests.json`,
then commit the result. Tests that need the public internet (real DERP)
cannot run hermetically; they are listed in `excluded-tests.json` and
run via the `integration-legacy` workflow instead.
## Framework overview

32
integration/dockertestutil/execute.go
View file

@ -4,6 +4,8 @@ import (
"bytes"
"errors"
"fmt"
"os"
"strconv"
"sync"
"time"
@ -11,15 +13,31 @@ import (
"github.com/ory/dockertest/v3"
)
// defaultExecuteTimeout returns the timeout for docker exec commands.
// On CI runners, docker exec latency is higher due to resource
// contention, so the timeout is doubled.
func defaultExecuteTimeout() time.Duration {
if util.IsCI() {
return 20 * time.Second
// ScaleTimeout scales an integration timeout for the running environment: the
// nested-docker nix-VM checks are slower than a host docker daemon and set
// HEADSCALE_INTEGRATION_TIMEOUT_SCALE to widen every budget; otherwise CI uses
// 2x and local dev the base. One knob covers exec, convergence, peer sync, and
// pool waits. Lives here (the lowest integration package) so every layer can
// reach it.
func ScaleTimeout(base time.Duration) time.Duration {
scale := os.Getenv("HEADSCALE_INTEGRATION_TIMEOUT_SCALE")
if scale != "" {
f, err := strconv.ParseFloat(scale, 64)
if err == nil && f > 0 {
return time.Duration(float64(base) * f)
}
}
return 10 * time.Second
if util.IsCI() {
return base * 2
}
return base
}
// defaultExecuteTimeout returns the timeout for docker exec commands.
func defaultExecuteTimeout() time.Duration {
return ScaleTimeout(10 * time.Second)
}
var (

79
integration/dsic/dsic.go
View file

@ -117,9 +117,14 @@ func (dsic *DERPServerInContainer) buildEntrypoint(derperArgs string) []string {
// Wait for network to be ready
commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done")
// Wait for TLS cert to be written (always written after container start)
// Wait for the TLS cert AND key to be written (both written after container
// start). Waiting only for the cert races: once it lands derper starts, and
// since it also needs the key — written a moment later — it exits and kills
// the container before the key upload (reliably, with slow nested-docker exec).
commands = append(commands,
fmt.Sprintf("while [ ! -f %s/%s.crt ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname))
commands = append(commands,
fmt.Sprintf("while [ ! -f %s/%s.key ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname))
// If CA certs are configured, wait for them to be written
if len(dsic.caCerts) > 0 {
@ -221,34 +226,56 @@ func New(
var container *dockertest.Resource
buildOptions := &dockertest.BuildOptions{
Dockerfile: "Dockerfile.derper",
ContextDir: dockerContextPath,
BuildArgs: []docker.BuildArg{},
}
switch version {
case "head":
buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
Name: "VERSION_BRANCH",
Value: "main",
})
default:
buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
Name: "VERSION_BRANCH",
Value: "v" + version,
})
}
// Add integration test labels if running under hi tool
dockertestutil.DockerAddIntegrationLabels(runOptions, "derp")
container, err = pool.BuildAndRunWithBuildOptions(
buildOptions,
runOptions,
dockertestutil.DockerRestartPolicy,
dockertestutil.DockerAllowLocalIPv6,
dockertestutil.DockerAllowNetworkAdministration,
)
// In CI / nix checks a prebuilt derper image is provided so we neither build
// the image nor clone tailscale at runtime, mirroring the tailscale and
// headscale image injection. Only the head image is prebuilt.
repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_DERPER_IMAGE")
if err != nil {
return nil, err
}
if prebuilt && version == "head" {
runOptions.Repository = repo
runOptions.Tag = tag
container, err = pool.RunWithOptions(
runOptions,
dockertestutil.DockerRestartPolicy,
dockertestutil.DockerAllowLocalIPv6,
dockertestutil.DockerAllowNetworkAdministration,
)
} else {
buildOptions := &dockertest.BuildOptions{
Dockerfile: "Dockerfile.derper",
ContextDir: dockerContextPath,
BuildArgs: []docker.BuildArg{},
}
switch version {
case "head":
buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
Name: "VERSION_BRANCH",
Value: "main",
})
default:
buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
Name: "VERSION_BRANCH",
Value: "v" + version,
})
}
container, err = pool.BuildAndRunWithBuildOptions(
buildOptions,
runOptions,
dockertestutil.DockerRestartPolicy,
dockertestutil.DockerAllowLocalIPv6,
dockertestutil.DockerAllowNetworkAdministration,
)
}
if err != nil {
return nil, fmt.Errorf(
"%s starting tailscale DERPer container (version: %s): %w",

7
integration/excluded-tests.json Normal file
View file

@ -0,0 +1,7 @@
[
"TestHASubnetRouterFailoverBothOfflineCablePull",
"TestHASubnetRouterFailoverDockerDisconnect",
"TestHASubnetRouterPingFailover",
"TestPingAllByIPPublicDERP",
"TestTailscaleRustAxum"
]

11
integration/general_test.go
View file

@ -483,16 +483,7 @@ func TestTaildrop(t *testing.T) {
_, err = scenario.ListTailscaleClientsFQDNs()
requireNoErrListFQDN(t, err)
// Install curl on all clients
for _, client := range allClients {
if !strings.Contains(client.Hostname(), "head") {
command := []string{"apk", "add", "curl"}
_, _, err := client.Execute(command)
if err != nil {
t.Fatalf("failed to install curl on %s, err: %s", client.Hostname(), err)
}
}
}
// curl is baked into every image (no runtime install).
// Helper to get FileTargets for a client.
getFileTargets := func(client TailscaleClient) ([]apitype.FileTarget, error) {

9
integration/helpers.go
View file

@ -22,6 +22,7 @@ import (
policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/juanfont/headscale/integration/integrationutil"
"github.com/juanfont/headscale/integration/tsic"
"github.com/oauth2-proxy/mockoidc"
@ -922,13 +923,7 @@ func assertCommandOutputContains(t *testing.T, c TailscaleClient, command []stri
// Uses longer timeouts in CI environments to account for slower resource allocation
// and higher system load during automated testing.
func dockertestMaxWait() time.Duration {
wait := 300 * time.Second //nolint
if util.IsCI() {
wait = 600 * time.Second //nolint
}
return wait
return dockertestutil.ScaleTimeout(300 * time.Second)
}
// didClientUseWebsocketForDERP analyzes client logs to determine if WebSocket was used for DERP.

33
integration/hsic/hsic.go
View file

@ -57,10 +57,8 @@ const (
)
var (
errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok")
errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag")
errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI")
errInvalidPostgresImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_POSTGRES_IMAGE format, expected repository:tag")
errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok")
errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI")
)
type fileInContainer struct {
@ -406,12 +404,12 @@ func New(
pgRepo := "postgres"
pgTag := "latest"
if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_POSTGRES_IMAGE"); prebuiltImage != "" {
repo, tag, found := strings.Cut(prebuiltImage, ":")
if !found {
return nil, errInvalidPostgresImageFormat
}
repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_POSTGRES_IMAGE")
if err != nil {
return nil, err
}
if prebuilt {
pgRepo = repo
pgTag = tag
}
@ -510,16 +508,13 @@ func New(
var container *dockertest.Resource
// Check if a pre-built image is available via environment variable
prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE")
if prebuiltImage != "" {
log.Printf("Using pre-built headscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value
// Parse image into repository and tag
repo, tag, ok := strings.Cut(prebuiltImage, ":")
if !ok {
return nil, errInvalidHeadscaleImageFormat
}
repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE")
if err != nil {
return nil, err
}
if prebuilt {
log.Printf("Using pre-built headscale image: %s:%s", repo, tag) //nolint:gosec // G706: integration-only log of trusted env value
runOptions.Repository = repo
runOptions.Tag = tag
@ -530,7 +525,7 @@ func New(
dockertestutil.DockerAllowNetworkAdministration,
)
if err != nil {
return nil, fmt.Errorf("running pre-built headscale container %q: %w", prebuiltImage, err)
return nil, fmt.Errorf("running pre-built headscale container %s:%s: %w", repo, tag, err)
}
} else if util.IsCI() {
return nil, errHeadscaleImageRequiredInCI

59
integration/integrationutil/util.go
View file

@ -8,28 +8,61 @@ import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"io"
"math/big"
"path/filepath"
"strings"
"time"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
"tailscale.com/envknob"
"tailscale.com/tailcfg"
)
// PeerSyncTimeout returns the timeout for peer synchronization based on environment:
// 60s for dev, 120s for CI.
func PeerSyncTimeout() time.Duration {
if util.IsCI() {
return 120 * time.Second
var errInvalidImageFormat = errors.New("integration image env must be in repository:tag format")
// PrebuiltImage reads a HEADSCALE_INTEGRATION_*_IMAGE knob (the full var name,
// e.g. "HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") and splits it into repository
// and tag. The bool is false when the knob is unset — the suite then builds the
// image itself; it errors when the value is set but is not "repository:tag".
// This is the one place the prebuilt-image knobs (used by the CI / nix-check
// path) are read, via tailscale's envknob.
func PrebuiltImage(envVar string) (string, string, bool, error) {
image := envknob.String(envVar)
if image == "" {
return "", "", false, nil
}
return 60 * time.Second
repo, tag, err := ParseImageRef(image)
if err != nil {
return "", "", false, fmt.Errorf("%s=%w", envVar, err)
}
return repo, tag, true, nil
}
// ParseImageRef splits an already-resolved "repository:tag" image string into
// its parts. Use it where the image comes from somewhere other than a single
// knob (e.g. a websocket-tagged override chosen at runtime); for the common
// read-a-knob case use [PrebuiltImage].
func ParseImageRef(image string) (string, string, error) {
repo, tag, found := strings.Cut(image, ":")
if !found {
return "", "", fmt.Errorf("%q: %w", image, errInvalidImageFormat)
}
return repo, tag, nil
}
// PeerSyncTimeout returns the timeout for peer synchronization: 60s for dev,
// scaled for CI / the slow nix VM.
func PeerSyncTimeout() time.Duration {
return dockertestutil.ScaleTimeout(60 * time.Second)
}
// PeerSyncRetryInterval returns the retry interval for peer synchronization checks.
@ -37,16 +70,10 @@ func PeerSyncRetryInterval() time.Duration {
return 100 * time.Millisecond
}
// ScaledTimeout returns the given timeout, scaled for CI environments
// where resource contention causes slower state propagation.
// Uses a 2x multiplier, consistent with PeerSyncTimeout (60s/120s)
// and dockertestMaxWait (300s/600s).
// ScaledTimeout returns the given convergence budget, scaled for the running
// environment via [dockertestutil.ScaleTimeout].
func ScaledTimeout(d time.Duration) time.Duration {
if util.IsCI() {
return d * 2
}
return d
return dockertestutil.ScaleTimeout(d)
}
func WriteFileToContainer(

8
integration/postgres-tests.nix Normal file
View file

@ -0,0 +1,8 @@
# Generated by gh-action-integration-generator.go; do not edit.
[
"TestACLAllowUserDst"
"TestPingAllByIP"
"TestEphemeral2006DeletedTooQuickly"
"TestPingAllByIPManyUpDown"
"TestSubnetRouterMultiNetwork"
]

103
integration/scenario.go
View file

@ -3,6 +3,7 @@ package integration
import (
"context"
"crypto/tls"
_ "embed"
"encoding/json"
"errors"
"fmt"
@ -22,7 +23,6 @@ import (
"time"
clientv1 "github.com/juanfont/headscale/gen/client/v1"
"github.com/juanfont/headscale/hscontrol/capver"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/juanfont/headscale/integration/dsic"
@ -54,30 +54,38 @@ var (
errNoHeadscaleAvailable = errors.New("no headscale available")
errNoUserAvailable = errors.New("no user available")
errNoClientFound = errors.New("client not found")
// AllVersions represents a list of Tailscale versions the suite
// uses to test compatibility with the [ControlServer].
//
// The list contains two special cases, "head" and "unstable" which
// points to the current tip of Tailscale's main branch and the latest
// released unstable version.
//
// The rest of the version represents Tailscale versions that can be
// found in Tailscale's apt repository.
AllVersions = append([]string{"head", "unstable"}, capver.TailscaleLatestMajorMinor(capver.SupportedMajorMinorVersions, true)...)
// MustTestVersions is the minimum set of versions we should test.
// At the moment, this is arbitrarily chosen as:
//
// - Two unstable (HEAD and unstable)
// - Two latest versions
// - Two oldest supported version.
MustTestVersions = append(
AllVersions[0:4],
AllVersions[len(AllVersions)-2:]...,
)
)
// Regenerate tailscale-versions.json from capver + nix-prefetch-docker. Runs in
// the same `make generate` / `go generate ./...` pass as capver (and needs the
// nix dev shell); already-pinned tags are reused, so an unchanged set is a
// no-op. Refresh just the pins with `go generate ./integration/...`.
//go:generate go run ../tools/tailscale-versions
//go:embed tailscale-versions.json
var tailscaleVersionsJSON []byte
// MustTestVersions is the set of Tailscale versions the suite tests for
// compatibility with the [ControlServer]. It contains "head" (built from
// source) and "unstable", plus the latest and oldest supported releases. It is
// read from the generated tailscale-versions.json — the single source shared
// with the nix image pins (regenerate with `update-integration-images`, which
// derives the list from capver).
var MustTestVersions = mustTailscaleVersions()
func mustTailscaleVersions() []string {
var v struct {
Versions []string `json:"versions"`
}
err := json.Unmarshal(tailscaleVersionsJSON, &v)
if err != nil {
panic("parsing tailscale-versions.json: " + err.Error())
}
return v.Versions
}
// User represents a User in the [ControlServer] and a map of [TailscaleClient]'s
// associated with the User.
type User struct {
@ -1583,6 +1591,33 @@ const (
var errStatusCodeNotOK = errors.New("status code not OK")
// runHeadscaleImageContainer starts runOptions from the prebuilt headscale
// image when HEADSCALE_INTEGRATION_HEADSCALE_IMAGE is set, otherwise it builds
// Dockerfile.integration. The mock OIDC provider (headscale mockoidc) and the
// webservice (python3 -m http.server) both run on the headscale image, so they
// share the same prebuilt-or-build path hsic uses for headscale itself — which
// is what lets them come up offline (e.g. in the nix VM checks).
func (s *Scenario) runHeadscaleImageContainer(runOptions *dockertest.RunOptions) (*dockertest.Resource, error) {
repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE")
if err != nil {
return nil, err
}
if prebuilt {
runOptions.Repository = repo
runOptions.Tag = tag
return s.pool.RunWithOptions(runOptions, dockertestutil.DockerRestartPolicy)
}
buildOptions := &dockertest.BuildOptions{
Dockerfile: hsic.IntegrationTestDockerFileName,
ContextDir: dockerContextPath,
}
return s.pool.BuildAndRunWithBuildOptions(buildOptions, runOptions, dockertestutil.DockerRestartPolicy)
}
func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUser) error {
port, err := dockertestutil.RandomFreeHostPort()
if err != nil {
@ -1618,11 +1653,6 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse
},
}
headscaleBuildOptions := &dockertest.BuildOptions{
Dockerfile: hsic.IntegrationTestDockerFileName,
ContextDir: dockerContextPath,
}
err = s.pool.RemoveContainerByName(hostname)
if err != nil {
return err
@ -1633,11 +1663,7 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse
// Add integration test labels if running under hi tool
dockertestutil.DockerAddIntegrationLabels(mockOidcOptions, "oidc")
if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( //nolint:noinlineerr
headscaleBuildOptions,
mockOidcOptions,
dockertestutil.DockerRestartPolicy,
); err == nil {
if pmockoidc, err := s.runHeadscaleImageContainer(mockOidcOptions); err == nil { //nolint:noinlineerr
s.mockOIDC.r = pmockoidc
} else {
return err
@ -1722,16 +1748,7 @@ func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) {
// Add integration test labels if running under hi tool
dockertestutil.DockerAddIntegrationLabels(webOpts, "web")
webBOpts := &dockertest.BuildOptions{
Dockerfile: hsic.IntegrationTestDockerFileName,
ContextDir: dockerContextPath,
}
web, err := s.pool.BuildAndRunWithBuildOptions(
webBOpts,
webOpts,
dockertestutil.DockerRestartPolicy,
)
web, err := s.runHeadscaleImageContainer(webOpts)
if err != nil {
return nil, err
}

8
integration/ssh_test.go
View file

@ -47,7 +47,6 @@ func sshScenario(t *testing.T, policy *policyv2.Policy, testName string, clients
// to not configure DNS.
tsic.WithNetfilter("off"),
tsic.WithPackages("openssh"),
tsic.WithExtraCommands("adduser ssh-it-user"),
tsic.WithDockerWorkdir("/"),
},
hsic.WithACLPolicy(policy),
@ -445,7 +444,7 @@ func doSSHWithRetryAsUser(
peerFQDN, _ := peer.FQDN()
command := []string{
"/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1",
"ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1",
fmt.Sprintf("%s@%s", sshUser, peerFQDN),
"'hostname'",
}
@ -661,7 +660,7 @@ func doSSHCheckWithTimeout(
peerFQDN, _ := peer.FQDN()
command := []string{
"/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30",
"ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30",
fmt.Sprintf("%s@%s", "ssh-it-user", peerFQDN),
"'hostname'",
}
@ -935,7 +934,6 @@ func TestSSHOneUserToOneCheckModeOIDC(t *testing.T) {
tsic.WithSSH(),
tsic.WithNetfilter("off"),
tsic.WithPackages("openssh"),
tsic.WithExtraCommands("adduser ssh-it-user"),
tsic.WithDockerWorkdir("/"),
},
hsic.WithACLPolicy(sshCheckPolicy()),
@ -1035,7 +1033,6 @@ func TestSSHCheckModeUnapprovedTimeout(t *testing.T) {
tsic.WithSSH(),
tsic.WithNetfilter("off"),
tsic.WithPackages("openssh"),
tsic.WithExtraCommands("adduser ssh-it-user"),
tsic.WithDockerWorkdir("/"),
},
hsic.WithACLPolicy(sshCheckPolicy()),
@ -1607,7 +1604,6 @@ func TestSSHLocalpart(t *testing.T) {
tsic.WithSSH(),
tsic.WithNetfilter("off"),
tsic.WithPackages("openssh"),
tsic.WithExtraCommands("adduser user1", "adduser user2"),
tsic.WithDockerWorkdir("/"),
},
hsic.WithTestName("sshlocalpart"),

66
integration/tailscale-versions.json Normal file
View file

@ -0,0 +1,66 @@
{
"versions": [
"head",
"unstable",
"1.80",
"1.82",
"1.96",
"1.98"
],
"images": {
"unstable": {
"imageName": "tailscale/tailscale",
"finalImageTag": "unstable",
"imageDigest": "sha256:85d46a2b396edbc3c0b1cda1d4433831236aa64da32fc80af0897501b1ab9ae2",
"sha256": {
"amd64": "sha256-UYtRaPnnDgRNomEX5EGp6yf/ye1KB/5RwaU2KWNRLsU=",
"arm64": "sha256-OmK7uLVskXSgto+ywvGlnhG5AhXOhlRYz4hi2dvp/a4="
}
},
"v1.80": {
"imageName": "ghcr.io/tailscale/tailscale",
"finalImageTag": "v1.80",
"imageDigest": "sha256:27b6a3dc30d89e94113b0d481dae05f08934cf80bdce860041727a2a60959921",
"sha256": {
"amd64": "sha256-CGNdS2gdO7VSG1NNOBUTxF4oeIwnUO00AWxyzwhWshs=",
"arm64": "sha256-YgX5Yy2Ls7HKOAXhwNNOQ9fgzK9EZqiRc9Q1Vej9A38="
}
},
"v1.82": {
"imageName": "ghcr.io/tailscale/tailscale",
"finalImageTag": "v1.82",
"imageDigest": "sha256:d26fc9bb035b0559900cc6f23506f6b1ddab61a690ffab4f5d84feceb3de811e",
"sha256": {
"amd64": "sha256-46PK+HTq6WVawxXj9Pktc1zHHGce/cSNAKAWFNX5NYA=",
"arm64": "sha256-Ij6qPQgECR7AGti8NCoNg5VaUZq3dKjQreQTOz/dDHE="
}
},
"v1.96": {
"imageName": "ghcr.io/tailscale/tailscale",
"finalImageTag": "v1.96",
"imageDigest": "sha256:dbeff02d2337344b351afac203427218c4d0a06c43fc10a865184063498472a6",
"sha256": {
"amd64": "sha256-F2vR1IhVeBOVRILxWIQ1N8DCX6foWmp6B/BZbFRgf5Q=",
"arm64": "sha256-q+hCmoH7BIX1B4BPBC9HJRlB7KHxFH9fFB6RKCYeK3A="
}
},
"v1.98": {
"imageName": "ghcr.io/tailscale/tailscale",
"finalImageTag": "v1.98",
"imageDigest": "sha256:854b77123b9536adae2e97f5a5fdb1790ed03438b911ab7f07780155e0af6ce2",
"sha256": {
"amd64": "sha256-zmPhOuNhcnExtY8L1G2za28fHpltgcmcdvGSEWcPVaY=",
"arm64": "sha256-J37dagB9SFlhAPTWQShty6muTmU71KLFKhBwpUtnxHc="
}
}
},
"postgres": {
"imageName": "postgres",
"finalImageTag": "16",
"imageDigest": "sha256:287eced1f33b59ed265ed13a60d3680dd7646d70c4dc0e785f59a470ebc03eeb",
"sha256": {
"amd64": "sha256-NgSEkOyXG8tdX9qRInuTd0n0fbVDKlPkJZz7haqFNMo=",
"arm64": "sha256-gcIWSO7wg1HlRUafEN7X9KdqpMgnar6gRf5IT7BiOdY="
}
}
}

159
integration/tests.nix Normal file
View file

@ -0,0 +1,159 @@
# Generated by gh-action-integration-generator.go; do not edit.
[
"TestACLHostsInNetMapTable"
"TestACLAllowUser80Dst"
"TestACLDenyAllPort80"
"TestACLAllowUserDst"
"TestACLAllowStarDst"
"TestACLNamedHostsCanReachBySubnet"
"TestACLNamedHostsCanReach"
"TestACLDevice1CanAccessDevice2"
"TestPolicyUpdateWhileRunningWithCLIInDatabase"
"TestACLAutogroupMember"
"TestACLAutogroupTagged"
"TestACLAutogroupSelf"
"TestACLPolicyPropagationOverTime"
"TestACLTagPropagation"
"TestACLTagPropagationPortSpecific"
"TestACLGroupWithUnknownUser"
"TestACLGroupAfterUserDeletion"
"TestACLGroupDeletionExactReproduction"
"TestACLDynamicUnknownUserAddition"
"TestACLDynamicUnknownUserRemoval"
"TestAPIAuthenticationBypass"
"TestAPIAuthenticationBypassCurl"
"TestRemoteCLIAuthenticationBypass"
"TestCLIWithConfigAuthenticationBypass"
"TestAuthKeyLogoutAndReloginSameUser"
"TestAuthKeyLogoutAndReloginNewUser"
"TestAuthKeyLogoutAndReloginSameUserExpiredKey"
"TestAuthKeyDeleteKey"
"TestAuthKeyLogoutAndReloginRoutesPreserved"
"TestOIDCAuthenticationPingAll"
"TestOIDCExpireNodesBasedOnTokenExpiry"
"TestOIDC024UserCreation"
"TestOIDCAuthenticationWithPKCE"
"TestOIDCReloginSameNodeNewUser"
"TestOIDCFollowUpUrl"
"TestOIDCMultipleOpenedLoginUrls"
"TestOIDCReloginSameNodeSameUser"
"TestOIDCExpiryAfterRestart"
"TestOIDCACLPolicyOnJoin"
"TestOIDCReloginSameUserRoutesPreserved"
"TestAuthWebFlowAuthenticationPingAll"
"TestAuthWebFlowLogoutAndReloginSameUser"
"TestAuthWebFlowLogoutAndReloginNewUser"
"TestApiKeyCommand"
"TestApiKeyCommandValidation"
"TestAuthCommandValidation"
"TestNodeCommand"
"TestNodeExpireCommand"
"TestNodeRenameCommand"
"TestPreAuthKeyCorrectUserLoggedInCommand"
"TestTaggedNodesCLIOutput"
"TestNodeExpireFlagsCommand"
"TestNodeCommandValidation"
"TestNodeTagCommand"
"TestNodeRouteCommands"
"TestNodeBackfillIPsCommand"
"TestOAuthClientCommand"
"TestOAuthClientCommandValidation"
"TestPolicyCheckCommand"
"TestSSHTestsRejectFailingPolicy"
"TestPolicyCommand"
"TestPolicyBrokenConfigCommand"
"TestPreAuthKeyCommand"
"TestPreAuthKeyCommandWithoutExpiry"
"TestPreAuthKeyCommandReusableEphemeral"
"TestPreAuthKeyDeleteCommand"
"TestPreAuthKeyCommandValidation"
"TestServerInfoCommands"
"TestUserCommand"
"TestUserCreateCommand"
"TestUserCommandValidation"
"TestDERPVerifyEndpoint"
"TestResolveMagicDNS"
"TestResolveMagicDNSExtraRecordsPath"
"TestDERPServerScenario"
"TestDERPServerWebsocketScenario"
"TestPingAllByIP"
"TestEphemeral"
"TestEphemeralInAlternateTimezone"
"TestEphemeral2006DeletedTooQuickly"
"TestPingAllByHostname"
"TestTaildrop"
"TestUpdateHostnameFromClient"
"TestExpireNode"
"TestSetNodeExpiryInFuture"
"TestDisableNodeExpiry"
"TestNodeOnlineStatus"
"TestPingAllByIPManyUpDown"
"Test2118DeletingOnlineNodePanics"
"TestGrantCapRelay"
"TestGrantCapDrive"
"TestK8sOperator"
"TestEnablingRoutes"
"TestHASubnetRouterFailover"
"TestSubnetRouteACL"
"TestEnablingExitRoutes"
"TestExitRoutesWithAutogroupInternetACL"
"TestSubnetRouterMultiNetwork"
"TestSubnetRouterMultiNetworkExitNode"
"TestAutoApproveMultiNetwork/authkey-tag.*"
"TestAutoApproveMultiNetwork/authkey-user.*"
"TestAutoApproveMultiNetwork/authkey-group.*"
"TestAutoApproveMultiNetwork/webauth-tag.*"
"TestAutoApproveMultiNetwork/webauth-user.*"
"TestAutoApproveMultiNetwork/webauth-group.*"
"TestSubnetRouteACLFiltering"
"TestGrantViaSubnetSteering"
"TestHASubnetRouterFailoverBothOffline"
"TestHeadscale"
"TestTailscaleNodesJoiningHeadcale"
"TestSSHOneUserToAll"
"TestSSHMultipleUsersAllToAll"
"TestSSHNoSSHConfigured"
"TestSSHIsBlockedInACL"
"TestSSHUserOnlyIsolation"
"TestSSHAutogroupSelf"
"TestSSHOneUserToOneCheckModeCLI"
"TestSSHOneUserToOneCheckModeOIDC"
"TestSSHCheckModeUnapprovedTimeout"
"TestSSHCheckModeCheckPeriodCLI"
"TestSSHCheckModeAutoApprove"
"TestSSHCheckModeSessionLossReDelegates"
"TestSSHCheckModeNegativeCLI"
"TestSSHLocalpart"
"TestTagsAuthKeyWithTagRequestDifferentTag"
"TestTagsAuthKeyWithTagNoAdvertiseFlag"
"TestTagsAuthKeyWithTagCannotAddViaCLI"
"TestTagsAuthKeyWithTagCannotChangeViaCLI"
"TestTagsAuthKeyWithTagAdminOverrideReauthPreserves"
"TestTagsAuthKeyWithTagCLICannotModifyAdminTags"
"TestTagsAuthKeyWithoutTagCannotRequestTags"
"TestTagsAuthKeyWithoutTagRegisterNoTags"
"TestTagsAuthKeyWithoutTagCannotAddViaCLI"
"TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset"
"TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise"
"TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag"
"TestTagsUserLoginOwnedTagAtRegistration"
"TestTagsUserLoginNonExistentTagAtRegistration"
"TestTagsUserLoginUnownedTagAtRegistration"
"TestTagsUserLoginAddTagViaCLIReauth"
"TestTagsUserLoginRemoveTagViaCLIReauth"
"TestTagsUserLoginCLINoOpAfterAdminAssignment"
"TestTagsUserLoginCLICannotRemoveAdminTags"
"TestTagsAuthKeyWithTagRequestNonExistentTag"
"TestTagsAuthKeyWithTagRequestUnownedTag"
"TestTagsAuthKeyWithoutTagRequestNonExistentTag"
"TestTagsAuthKeyWithoutTagRequestUnownedTag"
"TestTagsAdminAPICannotSetNonExistentTag"
"TestTagsAdminAPICanSetUnownedTag"
"TestTagsAdminAPICannotRemoveAllTags"
"TestTagsIssue2978ReproTagReplacement"
"TestTagsAdminAPICannotSetInvalidFormat"
"TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags"
"TestTagsAuthKeyWithoutUserInheritsTags"
"TestTagsAuthKeyWithoutUserRejectsAdvertisedTags"
"TestTagsAuthKeyConvertToUserViaCLIRegister"
]

75
integration/tsic/tsic.go
View file

@ -26,6 +26,7 @@ import (
"github.com/juanfont/headscale/integration/integrationutil"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
"tailscale.com/envknob"
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
"tailscale.com/ipn/store/mem"
@ -65,7 +66,6 @@ var (
errTailscaleWrongPeerCount = errors.New("wrong peer count")
errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey")
errInvalidClientConfig = errors.New("verifiably invalid client config requested")
errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag")
errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version")
errContainerNotInitialized = errors.New("container not initialized")
errFQDNNotYetAvailable = errors.New("FQDN not yet available")
@ -244,20 +244,33 @@ func WithAcceptRoutes() Option {
}
}
// WithPackages specifies Alpine packages to install when the container starts.
// This requires internet access and uses `apk add`. Common packages:
// WithPackages declares the tools a test needs inside the container. The images
// bake these in (no runtime install), so this asserts they are present at
// startup and fails the container loudly if not. Common packages:
// - "python3" for HTTP server
// - "curl" for HTTP client
// - "bind-tools" for dig command
// - "iptables", "ip6tables" for firewall rules
// Note: Tests using this option require internet access and cannot use
// the built-in DERP server in offline mode.
// - "openssh" for the ssh client.
func WithPackages(packages ...string) Option {
return func(tsic *TailscaleInContainer) {
tsic.withPackages = append(tsic.withPackages, packages...)
}
}
// packageBinary maps an alpine package name to a representative binary the
// images must provide, so [WithPackages] can assert the binary is on PATH
// rather than installing the package at runtime.
func packageBinary(pkg string) string {
switch pkg {
case "openssh":
return "ssh"
case "bind-tools":
return "dig"
default:
return pkg
}
}
// WithWebserver starts a Python HTTP server on the specified port
// alongside tailscaled. This is useful for testing subnet routing
// and ACL connectivity. Automatically adds "python3" to packages if needed.
@ -285,20 +298,31 @@ func (t *TailscaleInContainer) buildEntrypoint() []string {
commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done")
// If CA certs are configured, wait for them to be written by the Go code
// (certs are written after container start via [TailscaleInContainer.WriteFile])
if len(t.caCerts) > 0 {
// (certs are written after container start via [TailscaleInContainer.WriteFile]).
// Wait for the LAST cert (highest index): they are written in order, so once
// user-{N-1}.crt exists every earlier one does too. Waiting only for user-0
// races update-ca-certificates ahead of later certs (e.g. the headscale CA at
// user-1 when a DERP cert occupies user-0), permanently skipping their trust.
if n := len(t.caCerts); n > 0 {
commands = append(commands,
fmt.Sprintf("while [ ! -f %s/user-0.crt ]; do sleep 0.1; done", caCertRoot))
fmt.Sprintf("while [ ! -f %s/user-%d.crt ]; do sleep 0.1; done", caCertRoot, n-1))
}
// Install packages if requested (requires internet access)
// The images bake in every tool the suite needs (nix/images.nix for the head
// and version images, the alpine base otherwise), so assert the requested
// packages are present rather than installing them — a hermetic/offline run
// has no package mirror. Fail the container loudly if one is missing so an
// image regression is obvious in its log.
packages := t.withPackages
if t.withWebserverPort > 0 && !slices.Contains(packages, "python3") {
packages = append(packages, "python3")
}
if len(packages) > 0 {
commands = append(commands, "apk add --no-cache "+strings.Join(packages, " "))
for _, pkg := range packages {
bin := packageBinary(pkg)
commands = append(commands, fmt.Sprintf(
"command -v %s >/dev/null 2>&1 || { echo 'tsic: required tool %s (package %s) missing from image' >&2; exit 1; }",
bin, bin, pkg))
}
// Update CA certificates
@ -427,24 +451,31 @@ func New(
switch version {
case VersionHead:
// Check if a pre-built image is available via environment variable
prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE")
prebuiltImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE")
// If custom build tags are required (e.g., for websocket DERP), we cannot use
// the pre-built image as it won't have the necessary code compiled in.
// If custom build tags are required (e.g., for websocket DERP), the
// default pre-built image won't have the necessary code compiled in. A
// websocket-tagged prebuilt image can be injected separately (CI / nix
// checks build one with ts_debug_websockets); otherwise fall back to
// building the image with the tags.
hasBuildTags := len(tsic.buildConfig.tags) > 0
if hasBuildTags && prebuiltImage != "" {
log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value
prebuiltImage, tsic.buildConfig.tags)
prebuiltImage = ""
wsImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE")
if tsic.withWebsocketDERP && wsImage != "" {
prebuiltImage = wsImage
} else {
log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value
prebuiltImage, tsic.buildConfig.tags)
prebuiltImage = ""
}
}
if prebuiltImage != "" {
log.Printf("Using pre-built tailscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value
// Parse image into repository and tag
repo, tag, ok := strings.Cut(prebuiltImage, ":")
if !ok {
return nil, errInvalidTailscaleImageFormat
repo, tag, err := integrationutil.ParseImageRef(prebuiltImage)
if err != nil {
return nil, err
}
tailscaleOptions.Repository = repo

10
integration/tsric/tsric.go
View file

@ -12,13 +12,13 @@ import (
"fmt"
"io"
"log"
"os"
"strings"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/juanfont/headscale/integration/integrationutil"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
"tailscale.com/envknob"
"tailscale.com/util/rands"
)
@ -35,7 +35,7 @@ const (
// getPrebuiltImage returns the pre-built tailscale-rs Docker image name if set.
func getPrebuiltImage() string {
return os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE")
return envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE")
}
// TailscaleRustInContainer runs the tailscale-rs axum example as an
@ -210,9 +210,9 @@ func New(
if prebuiltImage := getPrebuiltImage(); prebuiltImage != "" {
log.Printf("Using pre-built tailscale-rs image: %s", prebuiltImage)
repo, tag, ok := strings.Cut(prebuiltImage, ":")
if !ok {
return nil, fmt.Errorf("tsric: invalid image format %q, expected repository:tag", prebuiltImage) //nolint:err113
repo, tag, err := integrationutil.ParseImageRef(prebuiltImage)
if err != nil {
return nil, fmt.Errorf("tsric: %w", err)
}
runOptions.Repository = repo

236
nix/images.nix Normal file
View file

@ -0,0 +1,236 @@
# Nix-built + pinned container images for the integration tests.
#
# Built images (headscale, tailscale-HEAD, derper) replace
# Dockerfile.integration-ci / Dockerfile.tailscale-HEAD / Dockerfile.derper for
# the nix/flake-check path. Released tailscale versions + postgres are pinned
# pulls (nix/tailscale-versions.nix). Everything is loaded into the VM docker so
# the checks run the full version matrix offline; the Go test code is unchanged
# and consumes them via the same HEADSCALE_INTEGRATION_*_IMAGE env vars / repo
# tags the `hi` CI path already uses.
{ pkgs, buildGoModule, tailscaleSrc }:
let
lib = pkgs.lib;
# Single source of truth for the tailscale version matrix + pinned registry
# images, shared with the Go suite (MustTestVersions reads the same file).
# Regenerate with `update-integration-images`.
versions = builtins.fromJSON (builtins.readFile ../integration/tailscale-versions.json);
# amd64/arm64 follows the build system; the tailscale + postgres registry
# images are multi-arch so the imageDigest (the index) is shared and we select
# the per-arch nix hash. Run on whatever arch the runner is, no forced cross-build.
dockerArch = if pkgs.stdenv.hostPlatform.isAarch64 then "arm64" else "amd64";
# Same headscale build, but from the Go-only filtered source and a fixed
# version (not the commit rev) so the image is content-addressed: identical
# source → identical store path → shared-cache hit across commits.
headscale = pkgs.headscale.overrideAttrs (_: {
src = pkgs.headscale-go-src;
version = "integration";
});
# alpine's `update-ca-certificates` has no drop-in nixpkgs equivalent. The
# integration containers write the per-test headscale CA to
# /usr/local/share/ca-certificates/user-N.crt and then call
# update-ca-certificates so TLS to headscale is trusted (integration/tsic,
# integration/hsic). This shim reproduces exactly that: append those certs
# onto a writable bundle that Go reads via SSL_CERT_FILE.
#
# ponytail: minimal CA shim; swap for a real ca-certificates package only if a
# test needs the full openssl rehash dance.
caShim = pkgs.writeShellScriptBin "update-ca-certificates" ''
set -eu
bundle=/etc/ssl/certs/ca-certificates.crt
mkdir -p /etc/ssl/certs
if [ ! -s "$bundle" ]; then
cp ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt "$bundle"
fi
for f in /usr/local/share/ca-certificates/*.crt; do
[ -e "$f" ] || continue
cat "$f" >> "$bundle"
done
'';
# taildrive (TestGrantCapDrive) spawns its per-user fileserver by re-execing
# tailscaled under `su` (tailscale drive/driveimpl/remote_impl.go): tailscaled
# runs as root and sus to the share's user. With no `su` on PATH canSU() fails,
# the fallback hits assertNotRoot() while root, the fileserver never starts,
# and every share fails with "unable to determine address for share". The
# released alpine images ship busybox su; expose just that applet here so it
# does not shadow coreutils on the image PATH.
busyboxSu = pkgs.runCommand "busybox-su" { } ''
mkdir -p $out/bin
ln -s ${pkgs.busybox}/bin/busybox $out/bin/su
'';
# Tailscale HEAD client + daemon + derper, built once from the pinned source.
# Mirrors Dockerfile.tailscale-HEAD / Dockerfile.derper's `go install`.
tailscaleHead = buildGoModule {
pname = "tailscale-head";
version = "head";
src = tailscaleSrc;
# Tailscale commits its own Go module vendorHash to flakehashes.json; reuse
# it so `nix flake update tailscale-head` carries the matching hash and we
# never hand-maintain it. (The hash tracks go.mod, not our subPackages.)
vendorHash = (builtins.fromJSON (builtins.readFile "${tailscaleSrc}/flakehashes.json")).vendor.sri;
subPackages = [ "cmd/tailscale" "cmd/tailscaled" "cmd/containerboot" "cmd/derper" ];
# ts_debug_websockets so this one image also serves the websocket-DERP test
# (no separate build); the code path is inert without TS_DEBUG_DERP_WS_CLIENT.
tags = [ "ts_debug_websockets" ];
env.CGO_ENABLED = "0";
doCheck = false;
};
# Userland the integration suite shells out to inside the containers:
# sh/sleep/cat (coreutils), find (findutils), ps/pidof (procps),
# grep/sed/awk, bash. Missing any of these makes test Execute() calls fail.
baseUserland = with pkgs; [
bashInteractive
coreutils
findutils
procps
gnugrep
gnused
gawk
];
# Shared image scaffolding. Go reads /etc/ssl/certs/ca-certificates.crt via
# SSL_CERT_FILE (maintained by caShim at container start).
commonEnv = [
"PATH=/bin:/usr/bin:/usr/local/bin:/sbin"
"SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt"
];
# /var/lib/tailscale must exist and be writable, else tailscaled's default
# state path resolves empty and it exits with "--state is required"
# (tailscale paths/paths_unix.go stateFileUnix). derper-certs holds the
# manual cert dsic writes before starting derper.
mkWritableDirs = ''
mkdir -p etc/ssl/certs usr/local/share/ca-certificates usr/local/share/derper-certs etc/headscale tmp var/run var/lib/tailscale
'';
# Local login accounts the SSH integration tests map tailscale SSH sessions to.
# Pre-created in both images at build time (head: seeded passwd below; released:
# busybox adduser in withTestTools) so no container is mutated at startup.
sshTestUsers = [ "ssh-it-user" "user1" "user2" ];
passwdFile = pkgs.writeText "passwd" (''
root:x:0:0:root:/root:/bin/sh
nobody:x:65534:65534:nobody:/:/bin/sh
'' + lib.concatStrings (lib.imap0
(i: u: "${u}:x:${toString (1000 + i)}:${toString (1000 + i)}::/home/${u}:/bin/sh\n")
sshTestUsers));
groupFile = pkgs.writeText "group" (''
root:x:0:
nobody:x:65534:
'' + lib.concatStrings (lib.imap0 (i: u: "${u}:x:${toString (1000 + i)}:\n") sshTestUsers));
tailscaleImage = pkgs.dockerTools.buildLayeredImage {
name = "tailscale-head";
tag = "nix";
contents = [
tailscaleHead
caShim
pkgs.dockerTools.binSh # /bin/sh -> bash (tsic entrypoint is /bin/sh -c)
pkgs.iproute2 # ip
pkgs.iptables # iptables, ip6tables
pkgs.curl
pkgs.python3 # webserver tests
pkgs.dnsutils # dig/nslookup (dns tests apk-add bind-tools)
pkgs.openssh # ssh, for the SSH tests
pkgs.getent # tailscale SSH server execs `getent passwd <user>`
pkgs.hostname # SSH tests run `hostname` on the peer over the session
pkgs.traceroute # HA/route tests assert path via traceroute (head-only)
busyboxSu # taildrive sus to spawn its per-user fileserver (TestGrantCapDrive)
] ++ baseUserland;
# buildLayeredImage puts every package's bin on /bin, which is on PATH, so
# the suite resolves tailscale/tailscaled/derper/curl/ssh/... directly — no
# symlinks needed. Just the writable state dirs and a passwd with the login
# the SSH tests use (ssh-it-user, pre-created so no runtime adduser).
extraCommands = ''
${mkWritableDirs}
${lib.concatMapStringsSep "\n" (u: "mkdir -p home/${u}") sshTestUsers}
cp ${passwdFile} etc/passwd
cp ${groupFile} etc/group
'';
config.Env = commonEnv;
};
headscaleImage = pkgs.dockerTools.buildLayeredImage {
name = "headscale";
tag = "nix";
contents = [
headscale
caShim
pkgs.dockerTools.binSh # hsic entrypoint is /bin/bash -c
pkgs.iproute2 # ip
pkgs.jq
pkgs.sqlite
pkgs.python3
pkgs.curl
pkgs.dnsutils # dig
] ++ baseUserland;
extraCommands = ''
${mkWritableDirs}
mkdir -p usr/local/bin
ln -sf ${headscale}/bin/headscale usr/local/bin/headscale
'';
config.Env = commonEnv;
};
# The tools the suite used to `apk add` into the released alpine images at
# runtime (curl/ssh/dig/python3/getent). A nixosTest has no network, so we bake
# them in at build time instead — one buildEnv copied onto the image so they
# land on /bin (already on the alpine PATH), no per-tool symlinks. Only /bin is
# linked: overlaying /lib would shadow musl's loader and break alpine's
# tailscaled. The binaries carry their own glibc closure via RPATH, so they run
# on musl without it. Tests assert these are present rather than installing them.
testTools = pkgs.buildEnv {
name = "ts-test-tools";
paths = with pkgs; [ curl openssh dnsutils python3 getent ];
pathsToLink = [ "/bin" ];
};
# Pinned registry images, as docker-loadable tarballs that keep their
# repo:tag so the suite finds them locally (its own pull fails offline, then
# RunWithOptions uses the loaded image). The spec's sha256 is per-arch.
pullImage = spec: pkgs.dockerTools.pullImage {
imageName = spec.imageName;
finalImageName = spec.imageName;
finalImageTag = spec.finalImageTag;
imageDigest = spec.imageDigest;
os = "linux";
arch = dockerArch;
sha256 = spec.sha256.${dockerArch};
};
# Released tailscale images with the test tools layered on, keeping the exact
# registry repo:tag the suite requests, and the SSH login pre-created at build
# time (alpine's busybox adduser) so no container is mutated at startup —
# matching the head image's baked ssh-it-user.
withTestTools = spec: pkgs.dockerTools.buildImage {
name = spec.imageName;
tag = spec.finalImageTag;
fromImage = pullImage spec;
copyToRoot = testTools;
runAsRoot = ''
#!${pkgs.runtimeShell}
export PATH=/usr/sbin:/usr/bin:/sbin:/bin
${lib.concatMapStringsSep "\n" (u: "grep -q '^${u}:' /etc/passwd || adduser -D -s /bin/sh ${u}") sshTestUsers}
'';
};
tailscaleVersionImages = map withTestTools (builtins.attrValues versions.images);
postgresImage = pullImage versions.postgres;
in
{
inherit
tailscaleImage
headscaleImage
postgresImage
tailscaleVersionImages
tailscaleHead
caShim;
}

157
nix/tests/integration.nix Normal file
View file

@ -0,0 +1,157 @@
# Per-test integration check: a NixOS VM that boots docker, loads the nix-built
# + pinned images (full tailscale version matrix), and runs the prebuilt
# integration test binary filtered to a single test. Hermetic equivalent of one
# `go run ./cmd/hi run <TestName>` job. `postgres = true` produces the postgres
# variant (HEADSCALE_INTEGRATION_POSTGRES=1) for the postgres subset.
#
# The suite runs *inside* a container named headscale-test-suite-<runID> with
# the docker socket mounted (docker-out-of-docker): scenario.go attaches that
# container to each test network, and IntegrationSkip gates on /.dockerenv. We
# reuse the headscale image as the runner base and bind-mount the host nix store
# so the test binary's closure resolves.
{ name
, testFilter ? name
, pkgs
, headscaleImage
, tailscaleImage
, postgresImage
, tailscaleVersionImages
, testBin
, src
, postgres ? false
}:
let
lib = pkgs.lib;
# The test function name, dropping any "/subtest" filter on a split check, so
# the skip-guard below matches a whole-function skip without tripping on a
# legitimately-skipped subtest.
testFunc = builtins.head (lib.splitString "/" testFilter);
# >=6 chars and DNS-safe: tsic/hsic derive container hostnames from the last
# 6 chars of the run ID (runID[len-6:]), which panics on shorter ids.
runID = "nixcheck";
runnerName = "headscale-test-suite-${runID}";
# Images this check must have locally before the suite runs. Every test may
# fan out over the full tailscale version matrix (MustTestVersions); the head
# image doubles as the DERP server; the postgres variant adds postgres.
images =
[ headscaleImage tailscaleImage ]
++ tailscaleVersionImages
++ lib.optional postgres postgresImage;
# Loading ~8 images dominates wall time; do it in parallel (they share layers,
# so docker dedups) and fail if any load fails.
loadImages = pkgs.writeShellScript "load-integration-images" ''
set -u
pids=""
${lib.concatMapStringsSep "\n" (img: ''docker load -i ${img} & pids="$pids $!"'') images}
rc=0
for p in $pids; do wait "$p" || rc=1; done
exit "$rc"
'';
pgEnv = lib.optionalString postgres
"-e HEADSCALE_INTEGRATION_POSTGRES=1 -e HEADSCALE_INTEGRATION_POSTGRES_IMAGE=postgres:16 ";
in
{
name = "integration-${name}${lib.optionalString postgres "-pg"}";
nodes.machine = { ... }: {
virtualisation.docker.enable = true;
# Match the integration CI: Docker 29's default containerd snapshotter +
# overlayfs regresses image load/exec for these images; the classic
# overlay2 graph driver is the known-good path.
virtualisation.docker.daemon.settings = {
storage-driver = "overlay2";
features.containerd-snapshotter = false;
};
# tailscaled needs /dev/net/tun for its --tun=tsdev device.
boot.kernelModules = [ "tun" ];
# The VM's dhcpcd otherwise manages docker's bridges/veths: as containers
# and the cable-pull tests churn interfaces it deletes their routes and
# races libnetwork's gateway setup, surfacing as
# "API error (500): updating gateway endpoint: failed to set gateway:
# network is unreachable" on reconnect (TestHASubnetRouterFailover*Disconnect
# /CablePull) and intermittent container-startup flakes. Keep it off them.
networking.dhcpcd.denyInterfaces = [ "veth*" "br-*" "docker*" ];
# Trim the guest toward a tiny profile: this VM only needs to boot, run
# docker, and exec one test binary. Measured boot is 18.7s (1.7s kernel +
# 3.9s initrd + 13.1s userspace); the userspace critical chain is
# dhcpcd (6.1s) -> network-online.target -> docker.service (+3.5s), i.e. the
# remaining cost is functional (docker waits for the network), not fat to
# trim. documentation + timesyncd off shave the cheap, avoidable part.
#
# microvm.nix was evaluated against this and rejected: it still needs a full
# docker daemon + nested KVM, and boot is only ~10-15% of the 2-3 min
# image-load + test runtime, so a new flake input can't pay itself back. The
# one real lever left, dropping docker's network-online wait (~6s), sits
# right next to the dhcpcd interface races handled below and isn't worth
# destabilising the matrix for.
documentation.enable = false;
services.timesyncd.enable = false;
# Sized for the full version matrix (a test can spawn 12+ tailscale
# containers + headscale + the runner) while still fitting a standard
# 4-core/16 GB GitHub runner. Disk-backed docker (not tmpfs) keeps RAM
# headroom on the runner; the slim source copy below is the bigger win.
virtualisation.memorySize = 8192;
virtualisation.cores = 4;
virtualisation.diskSize = 12288;
};
testScript = ''
start_all()
machine.wait_for_unit("docker.service")
# Load all matrix images in parallel (store paths visible inside the VM).
machine.succeed("${loadImages}", timeout=600)
# Writable copy of just the integration package (the suite writes
# control_logs/ under CWD and reads testdata relative to it). Only this
# subtree is needed at runtime — the repo-root build context is only used
# for image builds, which we bypass with prebuilt images.
machine.succeed("mkdir -p /tmp/hs && cp -r ${src}/integration /tmp/hs/integration && chmod -R u+w /tmp/hs")
# hsic/tsic save artifacts (and tsic.Status writes <host>_status.json) under
# /tmp/control; cmd/hi mounts this dir. Without it every Status() errors on
# the write and the suite never observes NeedsLogin.
machine.succeed("mkdir -p /tmp/control")
# Run the suite inside a container (docker-out-of-docker), mirroring cmd/hi:
# - name matches scenario.go's testSuiteName so it can join test networks
# - docker socket mounted so sibling containers spawn on this daemon
# - host nix store mounted so the test binary's closure resolves
out = machine.succeed(
"docker run --rm --name ${runnerName} "
"-v /var/run/docker.sock:/var/run/docker.sock "
"-v /nix/store:/nix/store:ro "
"-v /tmp/hs:/tmp/hs -w /tmp/hs/integration "
"-v /tmp/control:/tmp/control "
"-e HEADSCALE_INTEGRATION_HEADSCALE_IMAGE=headscale:nix "
"-e HEADSCALE_INTEGRATION_TAILSCALE_IMAGE=tailscale-head:nix "
"-e HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE=tailscale-head:nix "
"-e HEADSCALE_INTEGRATION_DERPER_IMAGE=tailscale-head:nix "
"${pgEnv}"
"-e HEADSCALE_INTEGRATION_RUN_ID=${runID} "
"-e CI=1 "
# The nested-docker VM converges slower than a host docker daemon; widen
# the ScaledTimeout budgets (subnet-route convergence, HA cycles) past 2x.
"-e HEADSCALE_INTEGRATION_TIMEOUT_SCALE=4 "
"-e SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt "
"headscale:nix "
"${testBin}/bin/integration.test -test.run '^${testFilter}$' -test.v -test.timeout=20m",
timeout=1800,
)
print(out)
# A skipped or unmatched test still exits 0; guard against a false green.
assert "no tests to run" not in out, "filter ${testFilter} matched nothing"
assert "--- SKIP: ${testFunc} " not in out, "test ${testFunc} was skipped (not in container?)"
'';
}

217
tools/tailscale-versions/main.go Normal file
View file

@ -0,0 +1,217 @@
// Command tailscale-versions regenerates integration/tailscale-versions.json:
// the bare version list the suite tests (from capver) plus the per-arch
// registry pins (manifest digest + nix sha256) the offline nix checks load.
//
// It is wired as a //go:generate step in the integration package so a capver
// bump refreshes the pins in the same `make generate` pass as capver itself. It
// shells out to nix-prefetch-docker, so it runs in the nix dev shell — but only
// for tags that aren't already pinned: a routine generate with an unchanged
// version set makes no network call and rewrites nothing.
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"regexp"
"github.com/juanfont/headscale/hscontrol/capver"
)
// versionsJSONPath is relative to the integration package directory, which is
// the working directory go:generate runs this command from.
const versionsJSONPath = "tailscale-versions.json"
// genArches are the docker architectures pinned for every image. The registry
// images are multi-arch, so the imageDigest (the manifest index) is shared and
// only the nix sha256 differs per arch; pullImage selects by the build system.
var genArches = []string{"amd64", "arm64"}
var (
reImageDigest = regexp.MustCompile(`imageDigest = "([^"]+)"`)
reHash = regexp.MustCompile(`hash = "([^"]+)"`)
errPrefetchEmpty = errors.New("nix-prefetch-docker produced no digest/hash (rate limited? retry)")
)
// imageRef pins a multi-arch registry image for dockerTools.pullImage.
type imageRef struct {
ImageName string `json:"imageName"`
FinalImageTag string `json:"finalImageTag"`
ImageDigest string `json:"imageDigest"`
SHA256 map[string]string `json:"sha256"`
}
// complete reports whether the pin already has a digest and every arch hash, so
// it can be reused without re-hitting the registry.
func (r imageRef) complete() bool {
if r.ImageDigest == "" {
return false
}
for _, a := range genArches {
if r.SHA256[a] == "" {
return false
}
}
return true
}
// versionsFile is the single source of truth shared by the Go suite
// (MustTestVersions reads .versions) and the nix image pins (images.nix reads
// .images / .postgres). See integration/tailscale-versions.json.
type versionsFile struct {
Versions []string `json:"versions"`
Images map[string]imageRef `json:"images"`
Postgres imageRef `json:"postgres"`
}
func main() {
err := run()
if err != nil {
fmt.Fprintln(os.Stderr, "tailscale-versions:", err)
os.Exit(1)
}
}
func run() error {
_, must := capver.IntegrationVersions()
prev := readExisting()
out := versionsFile{Versions: must, Images: map[string]imageRef{}}
for _, v := range must {
// head is built from source, not pulled.
if v == "head" {
continue
}
repo, tag := "ghcr.io/tailscale/tailscale", "v"+v
if v == "unstable" {
repo, tag = "tailscale/tailscale", "unstable"
}
ref, err := pinOrReuse(prev.Images[tag], repo, tag)
if err != nil {
return err
}
out.Images[tag] = ref
}
postgres, err := pinOrReuse(prev.Postgres, "postgres", "16")
if err != nil {
return err
}
out.Postgres = postgres
return writeIfChanged(out)
}
// pinOrReuse keeps an already-complete pin so a routine generate makes no
// network call; only new or incomplete tags are prefetched. To deliberately
// refresh a mutable tag (unstable, postgres) delete its entry first.
func pinOrReuse(prev imageRef, repo, tag string) (imageRef, error) {
if prev.complete() {
return prev, nil
}
return pinImage(context.Background(), repo, tag)
}
// readExisting loads the current pins, or an empty file if absent/unparseable
// (a full regeneration).
func readExisting() versionsFile {
vf := versionsFile{Images: map[string]imageRef{}}
b, err := os.ReadFile(versionsJSONPath)
if err != nil {
return vf
}
err = json.Unmarshal(b, &vf)
if err != nil {
return versionsFile{Images: map[string]imageRef{}}
}
if vf.Images == nil {
vf.Images = map[string]imageRef{}
}
return vf
}
// writeIfChanged only touches the file when the rendered JSON differs, so a
// no-op generate leaves the working tree (and git) clean.
func writeIfChanged(out versionsFile) error {
b, err := json.MarshalIndent(out, "", " ")
if err != nil {
return fmt.Errorf("marshalling versions: %w", err)
}
b = append(b, '\n')
old, err := os.ReadFile(versionsJSONPath)
if err == nil && bytes.Equal(old, b) {
fmt.Printf("%s already up to date\n", versionsJSONPath)
return nil
}
err = os.WriteFile(versionsJSONPath, b, 0o600)
if err != nil {
return fmt.Errorf("writing %s: %w", versionsJSONPath, err)
}
fmt.Printf("wrote %s\n", versionsJSONPath)
return nil
}
// pinImage prefetches repo:tag for every arch via nix-prefetch-docker and
// returns the shared manifest digest plus per-arch nix hashes.
func pinImage(ctx context.Context, repo, tag string) (imageRef, error) {
ref := imageRef{ImageName: repo, FinalImageTag: tag, SHA256: map[string]string{}}
for _, arch := range genArches {
cmd := exec.CommandContext(ctx, "nix", "run", "nixpkgs#nix-prefetch-docker", "--",
"--image-name", repo, "--image-tag", tag, "--arch", arch, "--os", "linux")
var stdout bytes.Buffer
cmd.Stdout = &stdout
err := cmd.Run()
if err != nil {
return imageRef{}, fmt.Errorf("prefetch %s:%s (%s): %w", repo, tag, arch, err)
}
digest := firstSubmatch(reImageDigest, stdout.String())
hash := firstSubmatch(reHash, stdout.String())
if digest == "" || hash == "" {
return imageRef{}, fmt.Errorf("%s:%s (%s): %w", repo, tag, arch, errPrefetchEmpty)
}
ref.ImageDigest = digest
ref.SHA256[arch] = hash
}
return ref, nil
}
func firstSubmatch(re *regexp.Regexp, s string) string {
m := re.FindStringSubmatch(s)
if len(m) < 2 {
return ""
}
return m[1]
}