diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 81246706c..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Build - -on: - push: - branches: - - main - pull_request: - -concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -defaults: - run: - shell: nix develop --fallback --command bash -e {0} - -jobs: - build-nix: - runs-on: ubuntu-latest - permissions: write-all - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 2 - - name: Get changed files - id: changed-files - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - with: - filters: | - files: - - '*.nix' - - 'go.*' - - '**/*.go' - - 'integration_test/' - - 'config-example.yaml' - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - if: steps.changed-files.outputs.files == 'true' - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - if: steps.changed-files.outputs.files == 'true' - - - name: Check vendor hash - id: vendorhash - if: steps.changed-files.outputs.files == 'true' - run: | - go run ./cmd/vendorhash check | tee check-result - { - grep '^expected_sri=' check-result || true - grep '^actual_sri=' check-result || true - } >> "$GITHUB_OUTPUT" - - - name: Vendor hash diverging - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - if: failure() && steps.vendorhash.outcome == 'failure' - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - github.rest.pulls.createReviewComment({ - pull_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'Vendor hash in `flakehashes.json` is stale (was `${{ steps.vendorhash.outputs.expected_sri }}`, should be `${{ steps.vendorhash.outputs.actual_sri }}`). Run `go run ./cmd/vendorhash update` and commit the result.' - }) - - - name: Run nix build - if: steps.changed-files.outputs.files == 'true' - run: nix build --fallback - - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - if: steps.changed-files.outputs.files == 'true' - with: - name: headscale-linux - path: result/bin/headscale - build-cross: - runs-on: ubuntu-latest - strategy: - matrix: - env: - - "GOARCH=arm64 GOOS=linux" - - "GOARCH=amd64 GOOS=linux" - - "GOARCH=arm64 GOOS=darwin" - - "GOARCH=amd64 GOOS=darwin" - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - - name: Run go cross compile - env: - CGO_ENABLED: 0 - run: env ${{ matrix.env }} go build -o "headscale" - ./cmd/headscale - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: "headscale-${{ matrix.env }}" - path: "headscale" diff --git a/.github/workflows/check-tests.yaml b/.github/workflows/check-tests.yaml index b73011732..dd3637b22 100644 --- a/.github/workflows/check-tests.yaml +++ b/.github/workflows/check-tests.yaml @@ -37,9 +37,23 @@ jobs: if: steps.changed-files.outputs.files == 'true' run: | (cd .github/workflows && go generate) - git diff --exit-code .github/workflows/test-integration.yaml + git diff --exit-code integration/tests.nix integration/postgres-tests.nix integration/excluded-tests.json - name: Show missing tests if: failure() run: | - git diff .github/workflows/test-integration.yaml + git diff integration/tests.nix integration/postgres-tests.nix integration/excluded-tests.json + + # The pinned tailscale images must match the versions the suite requests + # (MustTestVersions); a capver bump that isn't mirrored here makes the + # offline integration checks fail at runtime. + - name: Check tailscale version pins match capver + if: steps.changed-files.outputs.files == 'true' + run: | + want=$(nix develop --command go run ./cmd/hi list-versions --set=must --exclude=head | tr ' ' '\n' | sort) + have=$(jq -r '.images | keys[]' integration/tailscale-versions.json | sort) + if [ "$want" != "$have" ]; then + echo "::error::integration/tailscale-versions.json is stale vs capver; run 'update-integration-images' in the dev shell" + diff <(echo "$want") <(echo "$have") || true + exit 1 + fi diff --git a/.github/workflows/gh-action-integration-generator.go b/.github/workflows/gh-action-integration-generator.go index 5bb8318bd..cc9c069c0 100644 --- a/.github/workflows/gh-action-integration-generator.go +++ b/.github/workflows/gh-action-integration-generator.go @@ -4,30 +4,49 @@ package main import ( "bytes" + "encoding/json" "fmt" "log" + "os" "os/exec" + "slices" "strings" ) -// testsToSplit defines tests that should be split into multiple CI jobs. -// Key is the test function name, value is a list of subtest prefixes. -// Each prefix becomes a separate CI job as "TestName/prefix". -// -// Example: [TestAutoApproveMultiNetwork] has subtests like: -// - TestAutoApproveMultiNetwork/authkey-tag-advertiseduringup-false-pol-database -// - TestAutoApproveMultiNetwork/webauth-user-advertiseduringup-true-pol-file -// -// Splitting by approver type (tag, user, group) creates 6 CI jobs with 4 tests each: -// - TestAutoApproveMultiNetwork/authkey-tag.* (4 tests) -// - TestAutoApproveMultiNetwork/authkey-user.* (4 tests) -// - TestAutoApproveMultiNetwork/authkey-group.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-tag.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-user.* (4 tests) -// - TestAutoApproveMultiNetwork/webauth-group.* (4 tests) -// -// This reduces load per CI job (4 tests instead of 12) to avoid infrastructure -// flakiness when running many sequential Docker-based integration tests. +// excludedFromNixChecks lists tests the hermetic nix-VM checks cannot run, with +// the reason. They still run via cmd/hi, where the missing resource exists. +var excludedFromNixChecks = map[string]string{ + // Verifies connectivity through Tailscale's *public* DERP relays + // (HEADSCALE_DERP_URLS=controlplane.tailscale.com). A hermetic offline VM + // has no internet and the nix sandbox forbids network. + "TestPingAllByIPPublicDERP": "requires public internet DERP", + + // Also routes DERP through Tailscale's public relays: tailscale-rs cannot + // validate the embedded DERP's self-signed cert (WithPublicDERP in the + // test), so it needs the same internet access PublicDERP does. + "TestTailscaleRustAxum": "requires public internet DERP (tailscale-rs)", + + // These drive `docker network disconnect`/`connect` to simulate cable pulls. + // In the VM's nested docker, libnetwork intermittently fails the reconnect + // ("failed to set gateway: network is unreachable" — a stale IPAM/endpoint + // reservation), which does not happen against a host docker daemon. They run + // reliably via cmd/hi in the legacy pipeline. + "TestHASubnetRouterFailoverDockerDisconnect": "docker reconnect unreliable in nested VM docker", + "TestHASubnetRouterFailoverBothOfflineCablePull": "docker reconnect unreliable in nested VM docker", + + // Ping-driven HA failover: after a router recovers it asserts router 2 stays + // primary and traffic still traceroutes through it. In the VM's nested docker + // that data-plane state never converges (a 4x ScaledTimeout budget did not + // help — behaviour, not timing); it holds against a host docker daemon. + "TestHASubnetRouterPingFailover": "ping-failover data plane doesn't converge in nested VM docker", +} + +// testsToSplit defines tests whose subtests are run as separate checks. Key is +// the test function, value is a list of subtest prefixes; each prefix becomes +// its own check as "TestName/prefix.*". TestAutoApproveMultiNetwork fans out +// over approver × policy-mode × advertise (24 subtests); running all of them in +// one nested-docker VM check overruns the timeout, so split by approver into +// 6 checks of 4 — fewer sequential docker scenarios per check. var testsToSplit = map[string][]string{ "TestAutoApproveMultiNetwork": { "authkey-tag", @@ -39,26 +58,32 @@ var testsToSplit = map[string][]string{ }, } -// expandTests takes a list of test names and expands any that need splitting -// into multiple subtest patterns. +// expandTests replaces any splittable test with one entry per subtest prefix. +// The ".*" lets the check's "^...$"-wrapped -test.run match the full subtest +// names (e.g. authkey-tag-advertiseduringup-false-pol-database). func expandTests(tests []string) []string { - var expanded []string + expanded := make([]string, 0, len(tests)) for _, test := range tests { - if prefixes, ok := testsToSplit[test]; ok { - // This test should be split into multiple jobs. - // We append ".*" to each prefix because the CI runner wraps patterns - // with ^...$ anchors. Without ".*", a pattern like "authkey$" wouldn't - // match "authkey-tag-advertiseduringup-false-pol-database". - for _, prefix := range prefixes { - expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix)) - } - } else { + prefixes, ok := testsToSplit[test] + if !ok { expanded = append(expanded, test) + continue + } + + for _, prefix := range prefixes { + expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix)) } } + return expanded } +// This generator enumerates the integration test functions and writes them as +// nix lists (integration/tests.nix and integration/postgres-tests.nix). The +// flake maps each entry to a NixOS-VM check (sqlite for all, plus a postgres +// variant for the subset) — see nix/tests/integration.nix. One check per test +// function; the GitHub workflow derives its matrix from these checks. + func findTests() []string { rgBin, err := exec.LookPath("rg") if err != nil { @@ -89,43 +114,41 @@ func findTests() []string { return tests } -func updateYAML(tests []string, jobName string, testPath string) { - testsForYq := fmt.Sprintf("[%s]", strings.Join(tests, ", ")) +// writeNixList writes the test function names as a nix list, consumed by +// flake.nix to generate one integration check per test. +func writeNixList(tests []string, nixPath string) { + var b strings.Builder + b.WriteString("# Generated by gh-action-integration-generator.go; do not edit.\n") + b.WriteString("[\n") + for _, test := range tests { + fmt.Fprintf(&b, " %q\n", test) + } + b.WriteString("]\n") - yqCommand := fmt.Sprintf( - "yq eval '.jobs.%s.strategy.matrix.test = %s' %s -i", - jobName, - testsForYq, - testPath, - ) - cmd := exec.Command("bash", "-c", yqCommand) - - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err := cmd.Run() - if err != nil { - log.Printf("stdout: %s", stdout.String()) - log.Printf("stderr: %s", stderr.String()) - log.Fatalf("failed to run yq command: %s", err) + if err := os.WriteFile(nixPath, []byte(b.String()), 0o644); err != nil { + log.Fatalf("failed to write %s: %s", nixPath, err) } - fmt.Printf("YAML file (%s) job %s updated successfully\n", testPath, jobName) + fmt.Printf("Nix list (%s) updated successfully\n", nixPath) } func main() { tests := findTests() - // Expand tests that should be split into multiple jobs - expandedTests := expandTests(tests) + tests = slices.DeleteFunc(tests, func(name string) bool { + if reason, ok := excludedFromNixChecks[name]; ok { + log.Printf("excluding %s from nix checks: %s", name, reason) + return true + } + return false + }) - quotedTests := make([]string, len(expandedTests)) - for i, test := range expandedTests { - quotedTests[i] = fmt.Sprintf("\"%s\"", test) - } + // Split heavy tests into per-subtest checks (after exclusion, so an excluded + // test is never expanded). + tests = expandTests(tests) - // Define selected tests for PostgreSQL + // Selected tests also run against PostgreSQL to verify database + // compatibility (a postgres-variant check per name). postgresTestNames := []string{ "TestACLAllowUserDst", "TestPingAllByIP", @@ -134,12 +157,30 @@ func main() { "TestSubnetRouterMultiNetwork", } - quotedPostgresTests := make([]string, len(postgresTestNames)) - for i, test := range postgresTestNames { - quotedPostgresTests[i] = fmt.Sprintf("\"%s\"", test) + // Tests excluded from the hermetic nix checks still need to run somewhere, + // so emit them for the legacy cmd/hi GitHub-Actions matrix (real internet). + excluded := make([]string, 0, len(excludedFromNixChecks)) + for name := range excludedFromNixChecks { + excluded = append(excluded, name) + } + slices.Sort(excluded) + + writeNixList(tests, "../../integration/tests.nix") + writeNixList(postgresTestNames, "../../integration/postgres-tests.nix") + writeJSONList(excluded, "../../integration/excluded-tests.json") +} + +// writeJSONList writes the names as a JSON array, consumed as a GitHub-Actions +// matrix by the legacy cmd/hi workflow. +func writeJSONList(names []string, path string) { + data, err := json.MarshalIndent(names, "", " ") + if err != nil { + log.Fatalf("marshalling %s: %s", path, err) } - // Update both SQLite and PostgreSQL job matrices - updateYAML(quotedTests, "sqlite", "./test-integration.yaml") - updateYAML(quotedPostgresTests, "postgres", "./test-integration.yaml") + if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil { + log.Fatalf("failed to write %s: %s", path, err) + } + + fmt.Printf("JSON list (%s) updated successfully\n", path) } diff --git a/.github/workflows/integration-test-template.yml b/.github/workflows/integration-test-template.yml deleted file mode 100644 index 590185c96..000000000 --- a/.github/workflows/integration-test-template.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: Integration Test Template - -on: - workflow_call: - inputs: - test: - required: true - type: string - postgres_flag: - required: false - type: string - default: "" - database_name: - required: true - type: string - -jobs: - test: - runs-on: ubuntu-24.04-arm - env: - # Github does not allow us to access secrets in pull requests, - # so this env var is used to check if we have the secret or not. - # If we have the secrets, meaning we are running on push in a fork, - # there might be secrets available for more debugging. - # If TS_OAUTH_CLIENT_ID and TS_OAUTH_SECRET is set, then the job - # will join a debug tailscale network, set up SSH and a tmux session. - # The SSH will be configured to use the SSH key of the Github user - # that triggered the build. - HAS_TAILSCALE_SECRET: ${{ secrets.TS_OAUTH_CLIENT_ID }} - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 2 - - name: Tailscale - if: ${{ env.HAS_TAILSCALE_SECRET }} - uses: tailscale/github-action@a392da0a182bba0e9613b6243ebd69529b1878aa # v4.1.0 - with: - oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} - oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} - tags: tag:gh - - name: Setup SSH server for Actor - if: ${{ env.HAS_TAILSCALE_SECRET }} - uses: alexellis/setup-sshd-actor@master - - name: Download headscale image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: headscale-image - path: /tmp/artifacts - - name: Download tailscale HEAD image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: tailscale-head-image - path: /tmp/artifacts - - name: Download tailscale released images - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: tailscale-released-images - path: /tmp/artifacts - - name: Download hi binary - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: hi-binary - path: /tmp/artifacts - - name: Download Go cache - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: go-cache - path: /tmp/artifacts - - name: Download postgres image - if: ${{ inputs.postgres_flag == '--postgres=1' }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: postgres-image - path: /tmp/artifacts - - name: Force overlay2 storage driver - run: | - sudo mkdir -p /etc/docker - echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker version - - name: Load br_netfilter for in-cluster service routing - if: inputs.test == 'TestK8sOperator' - # TestK8sOperator runs k3s in a container; without br_netfilter on the - # host, bridged pod-to-pod traffic skips kube-proxy's ClusterIP DNAT and - # in-cluster DNS (kube-dns) is unreachable. The module cannot be loaded - # from inside the unprivileged-module rancher/k3s image, so load it here. - run: sudo modprobe br_netfilter - - name: Login to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - if: env.DOCKERHUB_USERNAME != '' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: Load Docker images, Go cache, and prepare binary - run: | - gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load - gunzip -c /tmp/artifacts/tailscale-head-image.tar.gz | docker load - gunzip -c /tmp/artifacts/tailscale-released-images.tar.gz | docker load - if [ -f /tmp/artifacts/postgres-image.tar.gz ]; then - gunzip -c /tmp/artifacts/postgres-image.tar.gz | docker load - fi - chmod +x /tmp/artifacts/hi - docker images - # Extract Go cache to host directories for bind mounting - mkdir -p /tmp/go-cache - tar -xzf /tmp/artifacts/go-cache.tar.gz -C /tmp/go-cache - ls -la /tmp/go-cache/ /tmp/go-cache/.cache/ - - name: Run Integration Test - env: - HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:${{ github.sha }} - HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:${{ github.sha }} - HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }} - HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go - HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build - # Mirror the docker/login-action secrets into env so the - # dockertestutil.Credentials resolver picks them up directly - # (otherwise it falls back to parsing ~/.docker/config.json, - # which works but is one step further from the source). - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \ - --timeout=120m \ - ${{ inputs.postgres_flag }} - # Sanitize test name for artifact upload (replace invalid characters: " : < > | * ? \ / with -) - - name: Sanitize test name for artifacts - if: always() - id: sanitize - run: echo "name=${TEST_NAME//[\":<>|*?\\\/]/-}" >> $GITHUB_OUTPUT - env: - TEST_NAME: ${{ inputs.test }} - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - if: always() - with: - name: ${{ inputs.database_name }}-${{ steps.sanitize.outputs.name }}-logs - path: "control_logs/*/*.log" - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - if: always() - with: - name: ${{ inputs.database_name }}-${{ steps.sanitize.outputs.name }}-artifacts - path: control_logs/ - - name: Setup a blocking tmux session - if: ${{ env.HAS_TAILSCALE_SECRET }} - uses: alexellis/block-with-tmux-action@master diff --git a/.github/workflows/nix-checks.yml b/.github/workflows/nix-checks.yml deleted file mode 100644 index 03c4b8e8b..000000000 --- a/.github/workflows/nix-checks.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Nix Flake Checks - -on: - push: - branches: - - main - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -# Each job only runs `nix build .#checks..`; the check logic lives -# in flake.nix via the flake-checks library. The fileset-filtered checks hit the -# hestia cache when their inputs are unchanged, so no changed-files gating. -permissions: - contents: read - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: build - run: nix build -L .#checks.x86_64-linux.build - - gotest: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: gotest - run: nix build -L .#checks.x86_64-linux.gotest - - golangci-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: golangci-lint - run: nix build -L .#checks.x86_64-linux.golangci-lint - - formatting: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: formatting - run: nix build -L .#checks.x86_64-linux.formatting diff --git a/.github/workflows/nix-module-test.yml b/.github/workflows/nix-module-test.yml deleted file mode 100644 index 1a7fbcc85..000000000 --- a/.github/workflows/nix-module-test.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: NixOS Module Tests - -on: - push: - branches: - - main - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - nix-module-check: - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 2 - - - name: Get changed files - id: changed-files - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - with: - filters: | - nix: - - 'nix/**' - - 'flake.nix' - - 'flake.lock' - go: - - 'go.*' - - '**/*.go' - - 'cmd/**' - - 'hscontrol/**' - - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true' - - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true' - - - name: Run NixOS module tests - if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true' - run: | - echo "Running NixOS module integration test..." - nix build .#checks.x86_64-linux.headscale -L --fallback diff --git a/.github/workflows/test-integration-legacy.yaml b/.github/workflows/test-integration-legacy.yaml new file mode 100644 index 000000000..5a56142e6 --- /dev/null +++ b/.github/workflows/test-integration-legacy.yaml @@ -0,0 +1,75 @@ +name: integration-legacy + +on: + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +# A small number of integration tests cannot run as hermetic NixOS-VM checks +# because they need Tailscale's public DERP relays (real internet) — see +# excludedFromNixChecks in gh-action-integration-generator.go. They run here the +# legacy way, via cmd/hi against Dockerfile-built images, which also keeps the +# cmd/hi path and the integration Dockerfiles exercised in CI. The matrix is the +# generated integration/excluded-tests.json so this list has a single source. +jobs: + list: + runs-on: ubuntu-latest + outputs: + tests: ${{ steps.list.outputs.tests }} + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - id: list + run: echo "tests=$(jq -c . integration/excluded-tests.json)" >> "$GITHUB_OUTPUT" + + test: + needs: list + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test: ${{ fromJSON(needs.list.outputs.tests) }} + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main + - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main + + # Docker 29's default containerd snapshotter regresses these images; the + # classic overlay2 graph driver is the known-good path (mirrors the VM checks). + - name: Force overlay2 storage driver + run: | + sudo mkdir -p /etc/docker + echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + # cmd/hi requires prebuilt head images in CI (util.IsCI); released versions + # pull from the registry and tailscale-rs is built inline by tsric. + # Dockerfile.integration-ci COPYs a prebuilt headscale binary from the + # build context, so build it first (tailscale-HEAD builds from source). + - name: Build headscale + tailscale-HEAD images + run: | + nix develop --command bash -c 'CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale' + docker build --file Dockerfile.integration-ci --tag headscale:ci . + docker build --file Dockerfile.tailscale-HEAD --tag tailscale-head:ci . + + - name: Run ${{ matrix.test }} + env: + HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:ci + HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:ci + run: nix develop --command go run ./cmd/hi run "^${{ matrix.test }}$" --timeout=900s diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml deleted file mode 100644 index 974e55233..000000000 --- a/.github/workflows/test-integration.yaml +++ /dev/null @@ -1,406 +0,0 @@ -name: integration -# To debug locally on a branch, and when needing secrets -# change this to include `push` so the build is ran on -# the main repository. -on: [pull_request] -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true -defaults: - run: - shell: nix develop --fallback --command bash -e {0} -jobs: - # build: Builds binaries and Docker images once, uploads as artifacts for reuse. - # build-postgres: Pulls postgres image separately to avoid Docker Hub rate limits. - # build-tailscale-released: Pre-pulls released Tailscale images from ghcr.io - # so fork PRs (no DOCKERHUB_USERNAME secret) don't hit Docker Hub rate - # limits at test time. - # sqlite: Runs all integration tests with SQLite backend. - # postgres: Runs a subset of tests with PostgreSQL to verify database compatibility. - build: - runs-on: ubuntu-24.04-arm - outputs: - files-changed: ${{ steps.changed-files.outputs.files }} - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 2 - - name: Get changed files - id: changed-files - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - with: - filters: | - files: - - '*.nix' - - 'go.*' - - '**/*.go' - - 'integration/**' - - 'config-example.yaml' - - '.github/workflows/test-integration.yaml' - - '.github/workflows/integration-test-template.yml' - - 'Dockerfile.*' - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - if: steps.changed-files.outputs.files == 'true' - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - if: steps.changed-files.outputs.files == 'true' - - name: Build binaries and warm Go cache - if: steps.changed-files.outputs.files == 'true' - run: | - # Build all Go binaries in one nix shell to maximize cache reuse - go build -o hi ./cmd/hi - CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale - # Build integration test binary to warm the cache with all dependencies - go test -c ./integration -o /dev/null 2>/dev/null || true - - name: Upload hi binary - if: steps.changed-files.outputs.files == 'true' - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: hi-binary - path: hi - retention-days: 10 - - name: Package Go cache - if: steps.changed-files.outputs.files == 'true' - run: | - # Package Go module cache and build cache - tar -czf go-cache.tar.gz -C ~ go .cache/go-build - - name: Upload Go cache - if: steps.changed-files.outputs.files == 'true' - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: go-cache - path: go-cache.tar.gz - retention-days: 10 - - name: Force overlay2 storage driver - if: steps.changed-files.outputs.files == 'true' - run: | - # Docker 29 runner images default to overlayfs, which breaks - # docker build via Go SDK libraries and docker save/load - # tarball formats. overlay2 is the long-standing default. - # https://github.com/actions/runner-images/issues/13474 - sudo mkdir -p /etc/docker - echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker version - - name: Login to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - if: env.DOCKERHUB_USERNAME != '' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: Build headscale image - if: steps.changed-files.outputs.files == 'true' - run: | - docker build \ - --file Dockerfile.integration-ci \ - --tag headscale:${{ github.sha }} \ - . - docker save headscale:${{ github.sha }} | gzip > headscale-image.tar.gz - - name: Build tailscale HEAD image - if: steps.changed-files.outputs.files == 'true' - run: | - docker build \ - --file Dockerfile.tailscale-HEAD \ - --tag tailscale-head:${{ github.sha }} \ - . - docker save tailscale-head:${{ github.sha }} | gzip > tailscale-head-image.tar.gz - - name: Upload headscale image - if: steps.changed-files.outputs.files == 'true' - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: headscale-image - path: headscale-image.tar.gz - retention-days: 10 - - name: Upload tailscale HEAD image - if: steps.changed-files.outputs.files == 'true' - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: tailscale-head-image - path: tailscale-head-image.tar.gz - retention-days: 10 - build-postgres: - runs-on: ubuntu-24.04-arm - needs: build - if: needs.build.outputs.files-changed == 'true' - steps: - - name: Force overlay2 storage driver - shell: bash - run: | - sudo mkdir -p /etc/docker - echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker version - - name: Login to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - if: env.DOCKERHUB_USERNAME != '' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: Pull and save postgres image - shell: bash - run: | - docker pull postgres:latest - docker tag postgres:latest postgres:${{ github.sha }} - docker save postgres:${{ github.sha }} | gzip > postgres-image.tar.gz - - name: Upload postgres image - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: postgres-image - path: postgres-image.tar.gz - retention-days: 10 - build-tailscale-released: - runs-on: ubuntu-24.04-arm - needs: build - if: needs.build.outputs.files-changed == 'true' - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main - - uses: Mic92/hestia/action@ff07bb902a9968ac0c3d0e51d90a606662a375d8 # main - - name: Force overlay2 storage driver - run: | - sudo mkdir -p /etc/docker - echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker version - - name: Login to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - if: env.DOCKERHUB_USERNAME != '' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: List Tailscale versions to pre-pull - id: versions - run: | - versions=$(go run ./cmd/hi list-versions --set=must --exclude=head) - echo "versions=${versions}" >> "$GITHUB_OUTPUT" - echo "Pre-pulling: ${versions}" - - name: Pull Tailscale images - run: | - # Releases come from ghcr.io (anonymous, unmetered). The - # "unstable" floating tag on ghcr.io has been stale since 2022, - # so it still needs to come from Docker Hub. xargs -P 0 fans - # out one process per tag and returns non-zero if any pull - # fails. - refs="" - for v in ${{ steps.versions.outputs.versions }}; do - if [ "${v}" = "unstable" ]; then - refs="${refs} tailscale/tailscale:${v}" - else - refs="${refs} ghcr.io/tailscale/tailscale:${v}" - fi - done - echo "${refs}" | tr ' ' '\n' | grep -v '^$' \ - | xargs -P 0 -I{} docker pull "{}" - echo "REFS=${refs}" >> "$GITHUB_ENV" - - name: Save Tailscale images to tarball - run: | - # Single docker save with all refs: one consistent snapshot, no - # parallel-daemon race. - docker save ${REFS} | gzip > tailscale-released-images.tar.gz - ls -lh tailscale-released-images.tar.gz - - name: Upload Tailscale released images - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: tailscale-released-images - path: tailscale-released-images.tar.gz - retention-days: 10 - sqlite: - needs: [build, build-tailscale-released] - if: needs.build.outputs.files-changed == 'true' - strategy: - fail-fast: false - matrix: - test: - - TestACLHostsInNetMapTable - - TestACLAllowUser80Dst - - TestACLDenyAllPort80 - - TestACLAllowUserDst - - TestACLAllowStarDst - - TestACLNamedHostsCanReachBySubnet - - TestACLNamedHostsCanReach - - TestACLDevice1CanAccessDevice2 - - TestPolicyUpdateWhileRunningWithCLIInDatabase - - TestACLAutogroupMember - - TestACLAutogroupTagged - - TestACLAutogroupSelf - - TestACLPolicyPropagationOverTime - - TestACLTagPropagation - - TestACLTagPropagationPortSpecific - - TestACLGroupWithUnknownUser - - TestACLGroupAfterUserDeletion - - TestACLGroupDeletionExactReproduction - - TestACLDynamicUnknownUserAddition - - TestACLDynamicUnknownUserRemoval - - TestAPIAuthenticationBypass - - TestAPIAuthenticationBypassCurl - - TestRemoteCLIAuthenticationBypass - - TestCLIWithConfigAuthenticationBypass - - TestAuthKeyLogoutAndReloginSameUser - - TestAuthKeyLogoutAndReloginNewUser - - TestAuthKeyLogoutAndReloginSameUserExpiredKey - - TestAuthKeyDeleteKey - - TestAuthKeyLogoutAndReloginRoutesPreserved - - TestOIDCAuthenticationPingAll - - TestOIDCExpireNodesBasedOnTokenExpiry - - TestOIDC024UserCreation - - TestOIDCAuthenticationWithPKCE - - TestOIDCReloginSameNodeNewUser - - TestOIDCFollowUpUrl - - TestOIDCMultipleOpenedLoginUrls - - TestOIDCReloginSameNodeSameUser - - TestOIDCExpiryAfterRestart - - TestOIDCACLPolicyOnJoin - - TestOIDCReloginSameUserRoutesPreserved - - TestAuthWebFlowAuthenticationPingAll - - TestAuthWebFlowLogoutAndReloginSameUser - - TestAuthWebFlowLogoutAndReloginNewUser - - TestApiKeyCommand - - TestApiKeyCommandValidation - - TestAuthCommandValidation - - TestNodeCommand - - TestNodeExpireCommand - - TestNodeRenameCommand - - TestPreAuthKeyCorrectUserLoggedInCommand - - TestTaggedNodesCLIOutput - - TestNodeExpireFlagsCommand - - TestNodeCommandValidation - - TestNodeTagCommand - - TestNodeRouteCommands - - TestNodeBackfillIPsCommand - - TestOAuthClientCommand - - TestOAuthClientCommandValidation - - TestPolicyCheckCommand - - TestSSHTestsRejectFailingPolicy - - TestPolicyCommand - - TestPolicyBrokenConfigCommand - - TestPreAuthKeyCommand - - TestPreAuthKeyCommandWithoutExpiry - - TestPreAuthKeyCommandReusableEphemeral - - TestPreAuthKeyDeleteCommand - - TestPreAuthKeyCommandValidation - - TestServerInfoCommands - - TestUserCommand - - TestUserCreateCommand - - TestUserCommandValidation - - TestDERPVerifyEndpoint - - TestResolveMagicDNS - - TestResolveMagicDNSExtraRecordsPath - - TestDERPServerScenario - - TestDERPServerWebsocketScenario - - TestPingAllByIP - - TestPingAllByIPPublicDERP - - TestEphemeral - - TestEphemeralInAlternateTimezone - - TestEphemeral2006DeletedTooQuickly - - TestPingAllByHostname - - TestTaildrop - - TestUpdateHostnameFromClient - - TestExpireNode - - TestSetNodeExpiryInFuture - - TestDisableNodeExpiry - - TestNodeOnlineStatus - - TestPingAllByIPManyUpDown - - Test2118DeletingOnlineNodePanics - - TestGrantCapRelay - - TestGrantCapDrive - - TestK8sOperator - - TestEnablingRoutes - - TestHASubnetRouterFailover - - TestSubnetRouteACL - - TestEnablingExitRoutes - - TestExitRoutesWithAutogroupInternetACL - - TestSubnetRouterMultiNetwork - - TestSubnetRouterMultiNetworkExitNode - - TestAutoApproveMultiNetwork/authkey-tag.* - - TestAutoApproveMultiNetwork/authkey-user.* - - TestAutoApproveMultiNetwork/authkey-group.* - - TestAutoApproveMultiNetwork/webauth-tag.* - - TestAutoApproveMultiNetwork/webauth-user.* - - TestAutoApproveMultiNetwork/webauth-group.* - - TestSubnetRouteACLFiltering - - TestGrantViaSubnetSteering - - TestHASubnetRouterPingFailover - - TestHASubnetRouterFailoverBothOffline - - TestHASubnetRouterFailoverBothOfflineCablePull - - TestHASubnetRouterFailoverDockerDisconnect - - TestHeadscale - - TestTailscaleNodesJoiningHeadcale - - TestSSHOneUserToAll - - TestSSHMultipleUsersAllToAll - - TestSSHNoSSHConfigured - - TestSSHIsBlockedInACL - - TestSSHUserOnlyIsolation - - TestSSHAutogroupSelf - - TestSSHOneUserToOneCheckModeCLI - - TestSSHOneUserToOneCheckModeOIDC - - TestSSHCheckModeUnapprovedTimeout - - TestSSHCheckModeCheckPeriodCLI - - TestSSHCheckModeAutoApprove - - TestSSHCheckModeSessionLossReDelegates - - TestSSHCheckModeNegativeCLI - - TestSSHLocalpart - - TestTagsAuthKeyWithTagRequestDifferentTag - - TestTagsAuthKeyWithTagNoAdvertiseFlag - - TestTagsAuthKeyWithTagCannotAddViaCLI - - TestTagsAuthKeyWithTagCannotChangeViaCLI - - TestTagsAuthKeyWithTagAdminOverrideReauthPreserves - - TestTagsAuthKeyWithTagCLICannotModifyAdminTags - - TestTagsAuthKeyWithoutTagCannotRequestTags - - TestTagsAuthKeyWithoutTagRegisterNoTags - - TestTagsAuthKeyWithoutTagCannotAddViaCLI - - TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset - - TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise - - TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag - - TestTagsUserLoginOwnedTagAtRegistration - - TestTagsUserLoginNonExistentTagAtRegistration - - TestTagsUserLoginUnownedTagAtRegistration - - TestTagsUserLoginAddTagViaCLIReauth - - TestTagsUserLoginRemoveTagViaCLIReauth - - TestTagsUserLoginCLINoOpAfterAdminAssignment - - TestTagsUserLoginCLICannotRemoveAdminTags - - TestTagsAuthKeyWithTagRequestNonExistentTag - - TestTagsAuthKeyWithTagRequestUnownedTag - - TestTagsAuthKeyWithoutTagRequestNonExistentTag - - TestTagsAuthKeyWithoutTagRequestUnownedTag - - TestTagsAdminAPICannotSetNonExistentTag - - TestTagsAdminAPICanSetUnownedTag - - TestTagsAdminAPICannotRemoveAllTags - - TestTagsIssue2978ReproTagReplacement - - TestTagsAdminAPICannotSetInvalidFormat - - TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags - - TestTagsAuthKeyWithoutUserInheritsTags - - TestTagsAuthKeyWithoutUserRejectsAdvertisedTags - - TestTagsAuthKeyConvertToUserViaCLIRegister - - TestTailscaleRustAxum - uses: ./.github/workflows/integration-test-template.yml - secrets: inherit - with: - test: ${{ matrix.test }} - postgres_flag: "--postgres=0" - database_name: "sqlite" - postgres: - needs: [build, build-postgres, build-tailscale-released] - if: needs.build.outputs.files-changed == 'true' - strategy: - fail-fast: false - matrix: - test: - - TestACLAllowUserDst - - TestPingAllByIP - - TestEphemeral2006DeletedTooQuickly - - TestPingAllByIPManyUpDown - - TestSubnetRouterMultiNetwork - uses: ./.github/workflows/integration-test-template.yml - secrets: inherit - with: - test: ${{ matrix.test }} - postgres_flag: "--postgres=1" - database_name: "postgres" diff --git a/.prettierignore b/.prettierignore index 9d8aa3bc9..a8dd52973 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1,6 @@ .github/workflows/test-integration-v2* docs/ +# Generated by gh-action-integration-generator.go (its own canonical format). +integration/excluded-tests.json +# Generated by update-integration-images (jq output). +integration/tailscale-versions.json diff --git a/cmd/hi/listversions.go b/cmd/hi/listversions.go index c2162ad64..ece773c1d 100644 --- a/cmd/hi/listversions.go +++ b/cmd/hi/listversions.go @@ -30,9 +30,7 @@ var listVersionsConfig ListVersionsConfig // tags, releases get a "v" prefix so each entry can be appended to // "ghcr.io/tailscale/tailscale:" directly. func listVersions(env *command.Env) error { - release := capver.TailscaleLatestMajorMinor(capver.SupportedMajorMinorVersions, true) - all := append([]string{"head", "unstable"}, release...) - must := append(append([]string{}, all[0:4]...), all[len(all)-2:]...) + all, must := capver.IntegrationVersions() var versions []string diff --git a/flake.lock b/flake.lock index c723d287a..86f9ce4c7 100644 --- a/flake.lock +++ b/flake.lock @@ -22,6 +22,22 @@ "type": "github" } }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -74,11 +90,28 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1772736753, + "narHash": "sha256-au/m3+EuBLoSzWUCb64a/MZq6QUtOV8oC0D9tY2scPQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "917fec990948658ef1ccd07cef2a1ef060786846", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake-checks": "flake-checks", "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "tailscale-head": "tailscale-head" } }, "systems": { @@ -111,6 +144,41 @@ "type": "github" } }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "tailscale-head": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs_2", + "systems": "systems_3" + }, + "locked": { + "lastModified": 1782250635, + "narHash": "sha256-y4/EAApBPekjWz2cP/9RexTBkiSAXsdOHWnhm/no/4I=", + "owner": "tailscale", + "repo": "tailscale", + "rev": "d4f2917c1bfd27529ec7997d08a8f1aa9ea903f2", + "type": "github" + }, + "original": { + "owner": "tailscale", + "repo": "tailscale", + "type": "github" + } + }, "treefmt-nix": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index 96786def3..91cc9f8c8 100644 --- a/flake.nix +++ b/flake.nix @@ -9,10 +9,18 @@ # once it ships go_1_26 >= 1.26.4. nixpkgs.url = "github:NixOS/nixpkgs/staging-next-26.05"; flake-utils.url = "github:numtide/flake-utils"; + # Reusable Go flake checks (build/test/lint/format); CI runs them via # `nix build .#checks..` instead of bespoke per-tool steps. flake-checks.url = "github:kradalby/flake-checks"; flake-checks.inputs.nixpkgs.follows = "nixpkgs"; + + # Tailscale HEAD, built from source for the integration test client image. + # Bump with `nix flake update tailscale-head` to track the latest main. + # It is a flake; we still build our own derivation (we need derper + + # containerboot, which its default package omits) but read its committed + # vendorHash from flakehashes.json so the bump carries the hash. + tailscale-head.url = "github:tailscale/tailscale"; }; outputs = @@ -20,6 +28,7 @@ , nixpkgs , flake-utils , flake-checks + , tailscale-head , ... }: let @@ -39,8 +48,27 @@ # Go 1.26 builder; resolves to Go 1.26.4 from the pinned nixpkgs. buildGo = pkgs.buildGo126Module; vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri; + + # Go source with the non-Go scaffolding filtered out, so editing the + # nix integration harness, CI workflows, docs or packaging does not + # rebuild the (slow) integration test binary or container images. + # Only changes to actual Go source invalidate them. + goSrc = pkgs.lib.fileset.toSource { + root = ./.; + fileset = pkgs.lib.fileset.difference ./. (pkgs.lib.fileset.unions [ + ./nix + ./.github + ./docs + ./packaging + ./flake.nix + ./flake.lock + ./flakehashes.json + ]); + }; in { + headscale-go-src = goSrc; + headscale = buildGo { pname = "headscale"; version = headscaleVersion; @@ -71,6 +99,34 @@ subPackages = [ "cmd/hi" ]; }; + # The integration suite compiled into a single test binary, run later + # inside a per-test NixOS VM check (see nix/tests/integration.nix). + # Compiled once and shared by every integration check. + integration-test-bin = buildGo { + pname = "headscale-integration-test"; + # Fixed (not the commit rev) so the binary is content-addressed: + # identical Go source → identical store path → shared-cache hit + # across commits. The version string is irrelevant for a test binary. + version = "integration"; + src = goSrc; + + inherit vendorHash; + + doCheck = false; + + buildPhase = '' + runHook preBuild + go test -c -o integration.test ./integration + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + install -Dm755 integration.test $out/bin/integration.test + runHook postInstall + ''; + }; + # Build golangci-lint with stock Go 1.26 (upstream uses hardcoded Go # version); it does not build against the pinned 1.26.4. golangci-lint = buildGo rec { @@ -200,7 +256,12 @@ vendorHash = (builtins.fromJSON (builtins.readFile ./flakehashes.json)).vendor.sri; goPkg = pkgs.go_1_26; # //go:embed targets and test-read files outside the default whitelist. - embedDirs = [ ./hscontrol/assets ./hscontrol/db/schema.sql ./config-example.yaml ]; + embedDirs = [ + ./hscontrol/assets + ./hscontrol/db/schema.sql + ./config-example.yaml + ./integration/tailscale-versions.json + ]; extraSrc = [ ./hscontrol/testdata ./hscontrol/types/testdata @@ -239,6 +300,45 @@ fmtExclude = [ ./gen ./docs ]; }); }; + + # Nix-built container images for the integration tests, plus the + # per-test VM check factory. Linux-only (dockerTools + nixosTest). + integrationImages = import ./nix/images.nix { + inherit pkgs; + buildGoModule = pkgs.buildGo126Module; + tailscaleSrc = tailscale-head; + }; + + mkIntegrationCheck = { name, testFilter ? name, postgres ? false }: + pkgs.testers.nixosTest (import ./nix/tests/integration.nix { + inherit name testFilter pkgs postgres; + inherit (integrationImages) + headscaleImage tailscaleImage postgresImage tailscaleVersionImages; + testBin = pkgs.integration-test-bin; + # goSrc (not the full tree) so a check's result stays cached when + # only nix/CI/docs change — only real Go/integration changes (which + # also rebuild testBin) invalidate it. Maximises shared-cache reuse. + src = pkgs.headscale-go-src; + }); + + # One sqlite check per test (full matrix), plus a postgres variant for + # the postgres subset — exactly the sqlite+postgres jobs the generator + # produces for the GitHub workflow. + integrationChecks = + let + # Split-test entries (TestAutoApproveMultiNetwork/authkey-tag.*) carry + # / and * in their -test.run filter; sanitise those into a valid check + # name while keeping the raw filter for -test.run. + sanitize = builtins.replaceStrings [ "/" "." "*" ] [ "-" "" "" ]; + mkCheck = postgres: filter: + pkgs.lib.nameValuePair + "integration-${sanitize filter}${pkgs.lib.optionalString postgres "-pg"}" + (mkIntegrationCheck { name = sanitize filter; testFilter = filter; inherit postgres; }); + in + pkgs.lib.listToAttrs ( + map (mkCheck false) (import ./integration/tests.nix) + ++ map (mkCheck true) (import ./integration/postgres-tests.nix) + ); in { # `nix develop` @@ -259,6 +359,14 @@ cat go.mod | ${pkgs.ripgrep}/bin/rg "\t" | ${pkgs.ripgrep}/bin/rg -v indirect | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.findutils}/bin/xargs go get -u go mod tidy '') + + # Regenerate integration/tailscale-versions.json after a capver bump + # so the offline checks pin the tailscale images the suite requests. + # It is a //go:generate step in the integration package (runs with + # capver under `make generate`); this just scopes it to the pins. + (pkgs.writeShellScriptBin "update-integration-images" '' + exec go generate ./integration/... + '') ]; shellHook = '' @@ -272,6 +380,28 @@ inherit headscale; inherit headscale-docker; default = headscale; + } + // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { + headscale-integration-image = integrationImages.headscaleImage; + tailscale-integration-image = integrationImages.tailscaleImage; + postgres-integration-image = integrationImages.postgresImage; + inherit (pkgs) integration-test-bin; + + # All shared inputs every integration check pulls in: the test binary + # and every image. CI builds this once (a "prime" job) so it lands in + # the shared nix cache before the per-test matrix fans out — the + # matrix jobs then substitute it instead of each rebuilding. + integration-deps = pkgs.linkFarm "integration-deps" ( + [ + { name = "test-bin"; path = pkgs.integration-test-bin; } + { name = "headscale"; path = integrationImages.headscaleImage; } + { name = "tailscale-head"; path = integrationImages.tailscaleImage; } + { name = "postgres"; path = integrationImages.postgresImage; } + ] + ++ pkgs.lib.imap0 + (i: img: { name = "ts-version-${toString i}"; path = img; }) + integrationImages.tailscaleVersionImages + ); }; # `nix run` @@ -287,6 +417,10 @@ } # The Go build/test checks are gated to Linux: parts of the tree are # Linux-specific and the pure unit subset is validated by CI. - // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks; + // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux goChecks + # Integration checks only on x86_64-linux: nixosTest needs KVM and the + # pinned version images are amd64 (nix/tailscale-versions.nix). aarch64 + # would need arch-specific pins. + // pkgs.lib.optionalAttrs (system == "x86_64-linux") integrationChecks; }); } diff --git a/hscontrol/capver/capver.go b/hscontrol/capver/capver.go index a4c535e88..5ebe2ce85 100644 --- a/hscontrol/capver/capver.go +++ b/hscontrol/capver/capver.go @@ -84,3 +84,17 @@ func TailscaleLatestMajorMinor(n int, stripV bool) []string { return majorSl[len(majorSl)-n:] } + +// IntegrationVersions returns the bare Tailscale version tags the integration +// suite pins and tests — the full set and the "must" subset (head, unstable, +// the two newest and two oldest tracked releases, for boundary coverage). +// Releases are bare (1.80); callers that build image tags prepend "v". This is +// the one source the suite's version list, the drift guard (`hi list-versions`), +// and the tailscale-versions.json generator all derive from. +func IntegrationVersions() ([]string, []string) { + release := TailscaleLatestMajorMinor(SupportedMajorMinorVersions, true) + all := append([]string{"head", "unstable"}, release...) + must := append(append([]string{}, all[0:4]...), all[len(all)-2:]...) + + return all, must +} diff --git a/integration/README.md b/integration/README.md index 5511a113f..ce180246d 100644 --- a/integration/README.md +++ b/integration/README.md @@ -21,16 +21,19 @@ go run ./cmd/hi doctor go run ./cmd/hi run "TestPingAllByIP" ``` -Alternatively, [`act`](https://github.com/nektos/act) runs the GitHub -Actions workflow locally: +In CI each test also runs as its own hermetic NixOS-VM flake check. Run +one locally exactly the way CI does: ```bash -act pull_request -W .github/workflows/test-integration.yaml +nix build .#checks.x86_64-linux.integration-TestPingAllByIP ``` -Each test runs as a separate workflow on GitHub Actions. To add a new -test, run `go generate` inside `../cmd/gh-action-integration-generator/` -and commit the generated workflow file. +The check matrix is generated: after adding or renaming a test, run `go +generate` in `.github/workflows/` to refresh `integration/tests.nix`, +`integration/postgres-tests.nix`, and `integration/excluded-tests.json`, +then commit the result. Tests that need the public internet (real DERP) +cannot run hermetically; they are listed in `excluded-tests.json` and +run via the `integration-legacy` workflow instead. ## Framework overview diff --git a/integration/dockertestutil/execute.go b/integration/dockertestutil/execute.go index 79511e2e0..d49ba6455 100644 --- a/integration/dockertestutil/execute.go +++ b/integration/dockertestutil/execute.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "fmt" + "os" + "strconv" "sync" "time" @@ -11,15 +13,31 @@ import ( "github.com/ory/dockertest/v3" ) -// defaultExecuteTimeout returns the timeout for docker exec commands. -// On CI runners, docker exec latency is higher due to resource -// contention, so the timeout is doubled. -func defaultExecuteTimeout() time.Duration { - if util.IsCI() { - return 20 * time.Second +// ScaleTimeout scales an integration timeout for the running environment: the +// nested-docker nix-VM checks are slower than a host docker daemon and set +// HEADSCALE_INTEGRATION_TIMEOUT_SCALE to widen every budget; otherwise CI uses +// 2x and local dev the base. One knob covers exec, convergence, peer sync, and +// pool waits. Lives here (the lowest integration package) so every layer can +// reach it. +func ScaleTimeout(base time.Duration) time.Duration { + scale := os.Getenv("HEADSCALE_INTEGRATION_TIMEOUT_SCALE") + if scale != "" { + f, err := strconv.ParseFloat(scale, 64) + if err == nil && f > 0 { + return time.Duration(float64(base) * f) + } } - return 10 * time.Second + if util.IsCI() { + return base * 2 + } + + return base +} + +// defaultExecuteTimeout returns the timeout for docker exec commands. +func defaultExecuteTimeout() time.Duration { + return ScaleTimeout(10 * time.Second) } var ( diff --git a/integration/dsic/dsic.go b/integration/dsic/dsic.go index c165eee10..3d531fa7e 100644 --- a/integration/dsic/dsic.go +++ b/integration/dsic/dsic.go @@ -117,9 +117,14 @@ func (dsic *DERPServerInContainer) buildEntrypoint(derperArgs string) []string { // Wait for network to be ready commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done") - // Wait for TLS cert to be written (always written after container start) + // Wait for the TLS cert AND key to be written (both written after container + // start). Waiting only for the cert races: once it lands derper starts, and + // since it also needs the key — written a moment later — it exits and kills + // the container before the key upload (reliably, with slow nested-docker exec). commands = append(commands, fmt.Sprintf("while [ ! -f %s/%s.crt ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname)) + commands = append(commands, + fmt.Sprintf("while [ ! -f %s/%s.key ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname)) // If CA certs are configured, wait for them to be written if len(dsic.caCerts) > 0 { @@ -221,34 +226,56 @@ func New( var container *dockertest.Resource - buildOptions := &dockertest.BuildOptions{ - Dockerfile: "Dockerfile.derper", - ContextDir: dockerContextPath, - BuildArgs: []docker.BuildArg{}, - } - - switch version { - case "head": - buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ - Name: "VERSION_BRANCH", - Value: "main", - }) - default: - buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ - Name: "VERSION_BRANCH", - Value: "v" + version, - }) - } // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(runOptions, "derp") - container, err = pool.BuildAndRunWithBuildOptions( - buildOptions, - runOptions, - dockertestutil.DockerRestartPolicy, - dockertestutil.DockerAllowLocalIPv6, - dockertestutil.DockerAllowNetworkAdministration, - ) + // In CI / nix checks a prebuilt derper image is provided so we neither build + // the image nor clone tailscale at runtime, mirroring the tailscale and + // headscale image injection. Only the head image is prebuilt. + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_DERPER_IMAGE") + if err != nil { + return nil, err + } + + if prebuilt && version == "head" { + runOptions.Repository = repo + runOptions.Tag = tag + + container, err = pool.RunWithOptions( + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + } else { + buildOptions := &dockertest.BuildOptions{ + Dockerfile: "Dockerfile.derper", + ContextDir: dockerContextPath, + BuildArgs: []docker.BuildArg{}, + } + + switch version { + case "head": + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "main", + }) + default: + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "v" + version, + }) + } + + container, err = pool.BuildAndRunWithBuildOptions( + buildOptions, + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + } + if err != nil { return nil, fmt.Errorf( "%s starting tailscale DERPer container (version: %s): %w", diff --git a/integration/excluded-tests.json b/integration/excluded-tests.json new file mode 100644 index 000000000..a9160c4da --- /dev/null +++ b/integration/excluded-tests.json @@ -0,0 +1,7 @@ +[ + "TestHASubnetRouterFailoverBothOfflineCablePull", + "TestHASubnetRouterFailoverDockerDisconnect", + "TestHASubnetRouterPingFailover", + "TestPingAllByIPPublicDERP", + "TestTailscaleRustAxum" +] diff --git a/integration/general_test.go b/integration/general_test.go index 9bdfd8c71..689c9a308 100644 --- a/integration/general_test.go +++ b/integration/general_test.go @@ -483,16 +483,7 @@ func TestTaildrop(t *testing.T) { _, err = scenario.ListTailscaleClientsFQDNs() requireNoErrListFQDN(t, err) - // Install curl on all clients - for _, client := range allClients { - if !strings.Contains(client.Hostname(), "head") { - command := []string{"apk", "add", "curl"} - _, _, err := client.Execute(command) - if err != nil { - t.Fatalf("failed to install curl on %s, err: %s", client.Hostname(), err) - } - } - } + // curl is baked into every image (no runtime install). // Helper to get FileTargets for a client. getFileTargets := func(client TailscaleClient) ([]apitype.FileTarget, error) { diff --git a/integration/helpers.go b/integration/helpers.go index 908c7447b..859d805fd 100644 --- a/integration/helpers.go +++ b/integration/helpers.go @@ -22,6 +22,7 @@ import ( policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/hscontrol/util" + "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/integrationutil" "github.com/juanfont/headscale/integration/tsic" "github.com/oauth2-proxy/mockoidc" @@ -922,13 +923,7 @@ func assertCommandOutputContains(t *testing.T, c TailscaleClient, command []stri // Uses longer timeouts in CI environments to account for slower resource allocation // and higher system load during automated testing. func dockertestMaxWait() time.Duration { - wait := 300 * time.Second //nolint - - if util.IsCI() { - wait = 600 * time.Second //nolint - } - - return wait + return dockertestutil.ScaleTimeout(300 * time.Second) } // didClientUseWebsocketForDERP analyzes client logs to determine if WebSocket was used for DERP. diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index d37bac9af..58de979c0 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -57,10 +57,8 @@ const ( ) var ( - errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") - errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag") - errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI") - errInvalidPostgresImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_POSTGRES_IMAGE format, expected repository:tag") + errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") + errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI") ) type fileInContainer struct { @@ -406,12 +404,12 @@ func New( pgRepo := "postgres" pgTag := "latest" - if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_POSTGRES_IMAGE"); prebuiltImage != "" { - repo, tag, found := strings.Cut(prebuiltImage, ":") - if !found { - return nil, errInvalidPostgresImageFormat - } + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_POSTGRES_IMAGE") + if err != nil { + return nil, err + } + if prebuilt { pgRepo = repo pgTag = tag } @@ -510,16 +508,13 @@ func New( var container *dockertest.Resource // Check if a pre-built image is available via environment variable - prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") - - if prebuiltImage != "" { - log.Printf("Using pre-built headscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value - // Parse image into repository and tag - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, errInvalidHeadscaleImageFormat - } + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") + if err != nil { + return nil, err + } + if prebuilt { + log.Printf("Using pre-built headscale image: %s:%s", repo, tag) //nolint:gosec // G706: integration-only log of trusted env value runOptions.Repository = repo runOptions.Tag = tag @@ -530,7 +525,7 @@ func New( dockertestutil.DockerAllowNetworkAdministration, ) if err != nil { - return nil, fmt.Errorf("running pre-built headscale container %q: %w", prebuiltImage, err) + return nil, fmt.Errorf("running pre-built headscale container %s:%s: %w", repo, tag, err) } } else if util.IsCI() { return nil, errHeadscaleImageRequiredInCI diff --git a/integration/integrationutil/util.go b/integration/integrationutil/util.go index a70145e0c..f31357c75 100644 --- a/integration/integrationutil/util.go +++ b/integration/integrationutil/util.go @@ -8,28 +8,61 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "fmt" "io" "math/big" "path/filepath" + "strings" "time" "github.com/juanfont/headscale/hscontrol/types" - "github.com/juanfont/headscale/hscontrol/util" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/tailcfg" ) -// PeerSyncTimeout returns the timeout for peer synchronization based on environment: -// 60s for dev, 120s for CI. -func PeerSyncTimeout() time.Duration { - if util.IsCI() { - return 120 * time.Second +var errInvalidImageFormat = errors.New("integration image env must be in repository:tag format") + +// PrebuiltImage reads a HEADSCALE_INTEGRATION_*_IMAGE knob (the full var name, +// e.g. "HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") and splits it into repository +// and tag. The bool is false when the knob is unset — the suite then builds the +// image itself; it errors when the value is set but is not "repository:tag". +// This is the one place the prebuilt-image knobs (used by the CI / nix-check +// path) are read, via tailscale's envknob. +func PrebuiltImage(envVar string) (string, string, bool, error) { + image := envknob.String(envVar) + if image == "" { + return "", "", false, nil } - return 60 * time.Second + repo, tag, err := ParseImageRef(image) + if err != nil { + return "", "", false, fmt.Errorf("%s=%w", envVar, err) + } + + return repo, tag, true, nil +} + +// ParseImageRef splits an already-resolved "repository:tag" image string into +// its parts. Use it where the image comes from somewhere other than a single +// knob (e.g. a websocket-tagged override chosen at runtime); for the common +// read-a-knob case use [PrebuiltImage]. +func ParseImageRef(image string) (string, string, error) { + repo, tag, found := strings.Cut(image, ":") + if !found { + return "", "", fmt.Errorf("%q: %w", image, errInvalidImageFormat) + } + + return repo, tag, nil +} + +// PeerSyncTimeout returns the timeout for peer synchronization: 60s for dev, +// scaled for CI / the slow nix VM. +func PeerSyncTimeout() time.Duration { + return dockertestutil.ScaleTimeout(60 * time.Second) } // PeerSyncRetryInterval returns the retry interval for peer synchronization checks. @@ -37,16 +70,10 @@ func PeerSyncRetryInterval() time.Duration { return 100 * time.Millisecond } -// ScaledTimeout returns the given timeout, scaled for CI environments -// where resource contention causes slower state propagation. -// Uses a 2x multiplier, consistent with PeerSyncTimeout (60s/120s) -// and dockertestMaxWait (300s/600s). +// ScaledTimeout returns the given convergence budget, scaled for the running +// environment via [dockertestutil.ScaleTimeout]. func ScaledTimeout(d time.Duration) time.Duration { - if util.IsCI() { - return d * 2 - } - - return d + return dockertestutil.ScaleTimeout(d) } func WriteFileToContainer( diff --git a/integration/postgres-tests.nix b/integration/postgres-tests.nix new file mode 100644 index 000000000..68d9a2228 --- /dev/null +++ b/integration/postgres-tests.nix @@ -0,0 +1,8 @@ +# Generated by gh-action-integration-generator.go; do not edit. +[ + "TestACLAllowUserDst" + "TestPingAllByIP" + "TestEphemeral2006DeletedTooQuickly" + "TestPingAllByIPManyUpDown" + "TestSubnetRouterMultiNetwork" +] diff --git a/integration/scenario.go b/integration/scenario.go index 7af281619..58cbeb72d 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -3,6 +3,7 @@ package integration import ( "context" "crypto/tls" + _ "embed" "encoding/json" "errors" "fmt" @@ -22,7 +23,6 @@ import ( "time" clientv1 "github.com/juanfont/headscale/gen/client/v1" - "github.com/juanfont/headscale/hscontrol/capver" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/dsic" @@ -54,30 +54,38 @@ var ( errNoHeadscaleAvailable = errors.New("no headscale available") errNoUserAvailable = errors.New("no user available") errNoClientFound = errors.New("client not found") - - // AllVersions represents a list of Tailscale versions the suite - // uses to test compatibility with the [ControlServer]. - // - // The list contains two special cases, "head" and "unstable" which - // points to the current tip of Tailscale's main branch and the latest - // released unstable version. - // - // The rest of the version represents Tailscale versions that can be - // found in Tailscale's apt repository. - AllVersions = append([]string{"head", "unstable"}, capver.TailscaleLatestMajorMinor(capver.SupportedMajorMinorVersions, true)...) - - // MustTestVersions is the minimum set of versions we should test. - // At the moment, this is arbitrarily chosen as: - // - // - Two unstable (HEAD and unstable) - // - Two latest versions - // - Two oldest supported version. - MustTestVersions = append( - AllVersions[0:4], - AllVersions[len(AllVersions)-2:]..., - ) ) +// Regenerate tailscale-versions.json from capver + nix-prefetch-docker. Runs in +// the same `make generate` / `go generate ./...` pass as capver (and needs the +// nix dev shell); already-pinned tags are reused, so an unchanged set is a +// no-op. Refresh just the pins with `go generate ./integration/...`. +//go:generate go run ../tools/tailscale-versions + +//go:embed tailscale-versions.json +var tailscaleVersionsJSON []byte + +// MustTestVersions is the set of Tailscale versions the suite tests for +// compatibility with the [ControlServer]. It contains "head" (built from +// source) and "unstable", plus the latest and oldest supported releases. It is +// read from the generated tailscale-versions.json — the single source shared +// with the nix image pins (regenerate with `update-integration-images`, which +// derives the list from capver). +var MustTestVersions = mustTailscaleVersions() + +func mustTailscaleVersions() []string { + var v struct { + Versions []string `json:"versions"` + } + + err := json.Unmarshal(tailscaleVersionsJSON, &v) + if err != nil { + panic("parsing tailscale-versions.json: " + err.Error()) + } + + return v.Versions +} + // User represents a User in the [ControlServer] and a map of [TailscaleClient]'s // associated with the User. type User struct { @@ -1583,6 +1591,33 @@ const ( var errStatusCodeNotOK = errors.New("status code not OK") +// runHeadscaleImageContainer starts runOptions from the prebuilt headscale +// image when HEADSCALE_INTEGRATION_HEADSCALE_IMAGE is set, otherwise it builds +// Dockerfile.integration. The mock OIDC provider (headscale mockoidc) and the +// webservice (python3 -m http.server) both run on the headscale image, so they +// share the same prebuilt-or-build path hsic uses for headscale itself — which +// is what lets them come up offline (e.g. in the nix VM checks). +func (s *Scenario) runHeadscaleImageContainer(runOptions *dockertest.RunOptions) (*dockertest.Resource, error) { + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") + if err != nil { + return nil, err + } + + if prebuilt { + runOptions.Repository = repo + runOptions.Tag = tag + + return s.pool.RunWithOptions(runOptions, dockertestutil.DockerRestartPolicy) + } + + buildOptions := &dockertest.BuildOptions{ + Dockerfile: hsic.IntegrationTestDockerFileName, + ContextDir: dockerContextPath, + } + + return s.pool.BuildAndRunWithBuildOptions(buildOptions, runOptions, dockertestutil.DockerRestartPolicy) +} + func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUser) error { port, err := dockertestutil.RandomFreeHostPort() if err != nil { @@ -1618,11 +1653,6 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse }, } - headscaleBuildOptions := &dockertest.BuildOptions{ - Dockerfile: hsic.IntegrationTestDockerFileName, - ContextDir: dockerContextPath, - } - err = s.pool.RemoveContainerByName(hostname) if err != nil { return err @@ -1633,11 +1663,7 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(mockOidcOptions, "oidc") - if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( //nolint:noinlineerr - headscaleBuildOptions, - mockOidcOptions, - dockertestutil.DockerRestartPolicy, - ); err == nil { + if pmockoidc, err := s.runHeadscaleImageContainer(mockOidcOptions); err == nil { //nolint:noinlineerr s.mockOIDC.r = pmockoidc } else { return err @@ -1722,16 +1748,7 @@ func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) { // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(webOpts, "web") - webBOpts := &dockertest.BuildOptions{ - Dockerfile: hsic.IntegrationTestDockerFileName, - ContextDir: dockerContextPath, - } - - web, err := s.pool.BuildAndRunWithBuildOptions( - webBOpts, - webOpts, - dockertestutil.DockerRestartPolicy, - ) + web, err := s.runHeadscaleImageContainer(webOpts) if err != nil { return nil, err } diff --git a/integration/ssh_test.go b/integration/ssh_test.go index 7ed50c206..fb14f8cd4 100644 --- a/integration/ssh_test.go +++ b/integration/ssh_test.go @@ -47,7 +47,6 @@ func sshScenario(t *testing.T, policy *policyv2.Policy, testName string, clients // to not configure DNS. tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(policy), @@ -445,7 +444,7 @@ func doSSHWithRetryAsUser( peerFQDN, _ := peer.FQDN() command := []string{ - "/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1", + "ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1", fmt.Sprintf("%s@%s", sshUser, peerFQDN), "'hostname'", } @@ -661,7 +660,7 @@ func doSSHCheckWithTimeout( peerFQDN, _ := peer.FQDN() command := []string{ - "/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30", + "ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30", fmt.Sprintf("%s@%s", "ssh-it-user", peerFQDN), "'hostname'", } @@ -935,7 +934,6 @@ func TestSSHOneUserToOneCheckModeOIDC(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(sshCheckPolicy()), @@ -1035,7 +1033,6 @@ func TestSSHCheckModeUnapprovedTimeout(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(sshCheckPolicy()), @@ -1607,7 +1604,6 @@ func TestSSHLocalpart(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser user1", "adduser user2"), tsic.WithDockerWorkdir("/"), }, hsic.WithTestName("sshlocalpart"), diff --git a/integration/tailscale-versions.json b/integration/tailscale-versions.json new file mode 100644 index 000000000..ca2b7a346 --- /dev/null +++ b/integration/tailscale-versions.json @@ -0,0 +1,66 @@ +{ + "versions": [ + "head", + "unstable", + "1.80", + "1.82", + "1.96", + "1.98" + ], + "images": { + "unstable": { + "imageName": "tailscale/tailscale", + "finalImageTag": "unstable", + "imageDigest": "sha256:85d46a2b396edbc3c0b1cda1d4433831236aa64da32fc80af0897501b1ab9ae2", + "sha256": { + "amd64": "sha256-UYtRaPnnDgRNomEX5EGp6yf/ye1KB/5RwaU2KWNRLsU=", + "arm64": "sha256-OmK7uLVskXSgto+ywvGlnhG5AhXOhlRYz4hi2dvp/a4=" + } + }, + "v1.80": { + "imageName": "ghcr.io/tailscale/tailscale", + "finalImageTag": "v1.80", + "imageDigest": "sha256:27b6a3dc30d89e94113b0d481dae05f08934cf80bdce860041727a2a60959921", + "sha256": { + "amd64": "sha256-CGNdS2gdO7VSG1NNOBUTxF4oeIwnUO00AWxyzwhWshs=", + "arm64": "sha256-YgX5Yy2Ls7HKOAXhwNNOQ9fgzK9EZqiRc9Q1Vej9A38=" + } + }, + "v1.82": { + "imageName": "ghcr.io/tailscale/tailscale", + "finalImageTag": "v1.82", + "imageDigest": "sha256:d26fc9bb035b0559900cc6f23506f6b1ddab61a690ffab4f5d84feceb3de811e", + "sha256": { + "amd64": "sha256-46PK+HTq6WVawxXj9Pktc1zHHGce/cSNAKAWFNX5NYA=", + "arm64": "sha256-Ij6qPQgECR7AGti8NCoNg5VaUZq3dKjQreQTOz/dDHE=" + } + }, + "v1.96": { + "imageName": "ghcr.io/tailscale/tailscale", + "finalImageTag": "v1.96", + "imageDigest": "sha256:dbeff02d2337344b351afac203427218c4d0a06c43fc10a865184063498472a6", + "sha256": { + "amd64": "sha256-F2vR1IhVeBOVRILxWIQ1N8DCX6foWmp6B/BZbFRgf5Q=", + "arm64": "sha256-q+hCmoH7BIX1B4BPBC9HJRlB7KHxFH9fFB6RKCYeK3A=" + } + }, + "v1.98": { + "imageName": "ghcr.io/tailscale/tailscale", + "finalImageTag": "v1.98", + "imageDigest": "sha256:854b77123b9536adae2e97f5a5fdb1790ed03438b911ab7f07780155e0af6ce2", + "sha256": { + "amd64": "sha256-zmPhOuNhcnExtY8L1G2za28fHpltgcmcdvGSEWcPVaY=", + "arm64": "sha256-J37dagB9SFlhAPTWQShty6muTmU71KLFKhBwpUtnxHc=" + } + } + }, + "postgres": { + "imageName": "postgres", + "finalImageTag": "16", + "imageDigest": "sha256:287eced1f33b59ed265ed13a60d3680dd7646d70c4dc0e785f59a470ebc03eeb", + "sha256": { + "amd64": "sha256-NgSEkOyXG8tdX9qRInuTd0n0fbVDKlPkJZz7haqFNMo=", + "arm64": "sha256-gcIWSO7wg1HlRUafEN7X9KdqpMgnar6gRf5IT7BiOdY=" + } + } +} diff --git a/integration/tests.nix b/integration/tests.nix new file mode 100644 index 000000000..5b14c095d --- /dev/null +++ b/integration/tests.nix @@ -0,0 +1,159 @@ +# Generated by gh-action-integration-generator.go; do not edit. +[ + "TestACLHostsInNetMapTable" + "TestACLAllowUser80Dst" + "TestACLDenyAllPort80" + "TestACLAllowUserDst" + "TestACLAllowStarDst" + "TestACLNamedHostsCanReachBySubnet" + "TestACLNamedHostsCanReach" + "TestACLDevice1CanAccessDevice2" + "TestPolicyUpdateWhileRunningWithCLIInDatabase" + "TestACLAutogroupMember" + "TestACLAutogroupTagged" + "TestACLAutogroupSelf" + "TestACLPolicyPropagationOverTime" + "TestACLTagPropagation" + "TestACLTagPropagationPortSpecific" + "TestACLGroupWithUnknownUser" + "TestACLGroupAfterUserDeletion" + "TestACLGroupDeletionExactReproduction" + "TestACLDynamicUnknownUserAddition" + "TestACLDynamicUnknownUserRemoval" + "TestAPIAuthenticationBypass" + "TestAPIAuthenticationBypassCurl" + "TestRemoteCLIAuthenticationBypass" + "TestCLIWithConfigAuthenticationBypass" + "TestAuthKeyLogoutAndReloginSameUser" + "TestAuthKeyLogoutAndReloginNewUser" + "TestAuthKeyLogoutAndReloginSameUserExpiredKey" + "TestAuthKeyDeleteKey" + "TestAuthKeyLogoutAndReloginRoutesPreserved" + "TestOIDCAuthenticationPingAll" + "TestOIDCExpireNodesBasedOnTokenExpiry" + "TestOIDC024UserCreation" + "TestOIDCAuthenticationWithPKCE" + "TestOIDCReloginSameNodeNewUser" + "TestOIDCFollowUpUrl" + "TestOIDCMultipleOpenedLoginUrls" + "TestOIDCReloginSameNodeSameUser" + "TestOIDCExpiryAfterRestart" + "TestOIDCACLPolicyOnJoin" + "TestOIDCReloginSameUserRoutesPreserved" + "TestAuthWebFlowAuthenticationPingAll" + "TestAuthWebFlowLogoutAndReloginSameUser" + "TestAuthWebFlowLogoutAndReloginNewUser" + "TestApiKeyCommand" + "TestApiKeyCommandValidation" + "TestAuthCommandValidation" + "TestNodeCommand" + "TestNodeExpireCommand" + "TestNodeRenameCommand" + "TestPreAuthKeyCorrectUserLoggedInCommand" + "TestTaggedNodesCLIOutput" + "TestNodeExpireFlagsCommand" + "TestNodeCommandValidation" + "TestNodeTagCommand" + "TestNodeRouteCommands" + "TestNodeBackfillIPsCommand" + "TestOAuthClientCommand" + "TestOAuthClientCommandValidation" + "TestPolicyCheckCommand" + "TestSSHTestsRejectFailingPolicy" + "TestPolicyCommand" + "TestPolicyBrokenConfigCommand" + "TestPreAuthKeyCommand" + "TestPreAuthKeyCommandWithoutExpiry" + "TestPreAuthKeyCommandReusableEphemeral" + "TestPreAuthKeyDeleteCommand" + "TestPreAuthKeyCommandValidation" + "TestServerInfoCommands" + "TestUserCommand" + "TestUserCreateCommand" + "TestUserCommandValidation" + "TestDERPVerifyEndpoint" + "TestResolveMagicDNS" + "TestResolveMagicDNSExtraRecordsPath" + "TestDERPServerScenario" + "TestDERPServerWebsocketScenario" + "TestPingAllByIP" + "TestEphemeral" + "TestEphemeralInAlternateTimezone" + "TestEphemeral2006DeletedTooQuickly" + "TestPingAllByHostname" + "TestTaildrop" + "TestUpdateHostnameFromClient" + "TestExpireNode" + "TestSetNodeExpiryInFuture" + "TestDisableNodeExpiry" + "TestNodeOnlineStatus" + "TestPingAllByIPManyUpDown" + "Test2118DeletingOnlineNodePanics" + "TestGrantCapRelay" + "TestGrantCapDrive" + "TestK8sOperator" + "TestEnablingRoutes" + "TestHASubnetRouterFailover" + "TestSubnetRouteACL" + "TestEnablingExitRoutes" + "TestExitRoutesWithAutogroupInternetACL" + "TestSubnetRouterMultiNetwork" + "TestSubnetRouterMultiNetworkExitNode" + "TestAutoApproveMultiNetwork/authkey-tag.*" + "TestAutoApproveMultiNetwork/authkey-user.*" + "TestAutoApproveMultiNetwork/authkey-group.*" + "TestAutoApproveMultiNetwork/webauth-tag.*" + "TestAutoApproveMultiNetwork/webauth-user.*" + "TestAutoApproveMultiNetwork/webauth-group.*" + "TestSubnetRouteACLFiltering" + "TestGrantViaSubnetSteering" + "TestHASubnetRouterFailoverBothOffline" + "TestHeadscale" + "TestTailscaleNodesJoiningHeadcale" + "TestSSHOneUserToAll" + "TestSSHMultipleUsersAllToAll" + "TestSSHNoSSHConfigured" + "TestSSHIsBlockedInACL" + "TestSSHUserOnlyIsolation" + "TestSSHAutogroupSelf" + "TestSSHOneUserToOneCheckModeCLI" + "TestSSHOneUserToOneCheckModeOIDC" + "TestSSHCheckModeUnapprovedTimeout" + "TestSSHCheckModeCheckPeriodCLI" + "TestSSHCheckModeAutoApprove" + "TestSSHCheckModeSessionLossReDelegates" + "TestSSHCheckModeNegativeCLI" + "TestSSHLocalpart" + "TestTagsAuthKeyWithTagRequestDifferentTag" + "TestTagsAuthKeyWithTagNoAdvertiseFlag" + "TestTagsAuthKeyWithTagCannotAddViaCLI" + "TestTagsAuthKeyWithTagCannotChangeViaCLI" + "TestTagsAuthKeyWithTagAdminOverrideReauthPreserves" + "TestTagsAuthKeyWithTagCLICannotModifyAdminTags" + "TestTagsAuthKeyWithoutTagCannotRequestTags" + "TestTagsAuthKeyWithoutTagRegisterNoTags" + "TestTagsAuthKeyWithoutTagCannotAddViaCLI" + "TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset" + "TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise" + "TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag" + "TestTagsUserLoginOwnedTagAtRegistration" + "TestTagsUserLoginNonExistentTagAtRegistration" + "TestTagsUserLoginUnownedTagAtRegistration" + "TestTagsUserLoginAddTagViaCLIReauth" + "TestTagsUserLoginRemoveTagViaCLIReauth" + "TestTagsUserLoginCLINoOpAfterAdminAssignment" + "TestTagsUserLoginCLICannotRemoveAdminTags" + "TestTagsAuthKeyWithTagRequestNonExistentTag" + "TestTagsAuthKeyWithTagRequestUnownedTag" + "TestTagsAuthKeyWithoutTagRequestNonExistentTag" + "TestTagsAuthKeyWithoutTagRequestUnownedTag" + "TestTagsAdminAPICannotSetNonExistentTag" + "TestTagsAdminAPICanSetUnownedTag" + "TestTagsAdminAPICannotRemoveAllTags" + "TestTagsIssue2978ReproTagReplacement" + "TestTagsAdminAPICannotSetInvalidFormat" + "TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags" + "TestTagsAuthKeyWithoutUserInheritsTags" + "TestTagsAuthKeyWithoutUserRejectsAdvertisedTags" + "TestTagsAuthKeyConvertToUserViaCLIRegister" +] diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 3e19d2b14..6770c837e 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -26,6 +26,7 @@ import ( "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store/mem" @@ -65,7 +66,6 @@ var ( errTailscaleWrongPeerCount = errors.New("wrong peer count") errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey") errInvalidClientConfig = errors.New("verifiably invalid client config requested") - errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag") errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version") errContainerNotInitialized = errors.New("container not initialized") errFQDNNotYetAvailable = errors.New("FQDN not yet available") @@ -244,20 +244,33 @@ func WithAcceptRoutes() Option { } } -// WithPackages specifies Alpine packages to install when the container starts. -// This requires internet access and uses `apk add`. Common packages: +// WithPackages declares the tools a test needs inside the container. The images +// bake these in (no runtime install), so this asserts they are present at +// startup and fails the container loudly if not. Common packages: // - "python3" for HTTP server // - "curl" for HTTP client // - "bind-tools" for dig command -// - "iptables", "ip6tables" for firewall rules -// Note: Tests using this option require internet access and cannot use -// the built-in DERP server in offline mode. +// - "openssh" for the ssh client. func WithPackages(packages ...string) Option { return func(tsic *TailscaleInContainer) { tsic.withPackages = append(tsic.withPackages, packages...) } } +// packageBinary maps an alpine package name to a representative binary the +// images must provide, so [WithPackages] can assert the binary is on PATH +// rather than installing the package at runtime. +func packageBinary(pkg string) string { + switch pkg { + case "openssh": + return "ssh" + case "bind-tools": + return "dig" + default: + return pkg + } +} + // WithWebserver starts a Python HTTP server on the specified port // alongside tailscaled. This is useful for testing subnet routing // and ACL connectivity. Automatically adds "python3" to packages if needed. @@ -285,20 +298,31 @@ func (t *TailscaleInContainer) buildEntrypoint() []string { commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done") // If CA certs are configured, wait for them to be written by the Go code - // (certs are written after container start via [TailscaleInContainer.WriteFile]) - if len(t.caCerts) > 0 { + // (certs are written after container start via [TailscaleInContainer.WriteFile]). + // Wait for the LAST cert (highest index): they are written in order, so once + // user-{N-1}.crt exists every earlier one does too. Waiting only for user-0 + // races update-ca-certificates ahead of later certs (e.g. the headscale CA at + // user-1 when a DERP cert occupies user-0), permanently skipping their trust. + if n := len(t.caCerts); n > 0 { commands = append(commands, - fmt.Sprintf("while [ ! -f %s/user-0.crt ]; do sleep 0.1; done", caCertRoot)) + fmt.Sprintf("while [ ! -f %s/user-%d.crt ]; do sleep 0.1; done", caCertRoot, n-1)) } - // Install packages if requested (requires internet access) + // The images bake in every tool the suite needs (nix/images.nix for the head + // and version images, the alpine base otherwise), so assert the requested + // packages are present rather than installing them — a hermetic/offline run + // has no package mirror. Fail the container loudly if one is missing so an + // image regression is obvious in its log. packages := t.withPackages if t.withWebserverPort > 0 && !slices.Contains(packages, "python3") { packages = append(packages, "python3") } - if len(packages) > 0 { - commands = append(commands, "apk add --no-cache "+strings.Join(packages, " ")) + for _, pkg := range packages { + bin := packageBinary(pkg) + commands = append(commands, fmt.Sprintf( + "command -v %s >/dev/null 2>&1 || { echo 'tsic: required tool %s (package %s) missing from image' >&2; exit 1; }", + bin, bin, pkg)) } // Update CA certificates @@ -427,24 +451,31 @@ func New( switch version { case VersionHead: // Check if a pre-built image is available via environment variable - prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE") + prebuiltImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE") - // If custom build tags are required (e.g., for websocket DERP), we cannot use - // the pre-built image as it won't have the necessary code compiled in. + // If custom build tags are required (e.g., for websocket DERP), the + // default pre-built image won't have the necessary code compiled in. A + // websocket-tagged prebuilt image can be injected separately (CI / nix + // checks build one with ts_debug_websockets); otherwise fall back to + // building the image with the tags. hasBuildTags := len(tsic.buildConfig.tags) > 0 if hasBuildTags && prebuiltImage != "" { - log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value - prebuiltImage, tsic.buildConfig.tags) - prebuiltImage = "" + wsImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE") + if tsic.withWebsocketDERP && wsImage != "" { + prebuiltImage = wsImage + } else { + log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value + prebuiltImage, tsic.buildConfig.tags) + prebuiltImage = "" + } } if prebuiltImage != "" { log.Printf("Using pre-built tailscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value - // Parse image into repository and tag - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, errInvalidTailscaleImageFormat + repo, tag, err := integrationutil.ParseImageRef(prebuiltImage) + if err != nil { + return nil, err } tailscaleOptions.Repository = repo diff --git a/integration/tsric/tsric.go b/integration/tsric/tsric.go index e160d855d..fdeef4942 100644 --- a/integration/tsric/tsric.go +++ b/integration/tsric/tsric.go @@ -12,13 +12,13 @@ import ( "fmt" "io" "log" - "os" "strings" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/util/rands" ) @@ -35,7 +35,7 @@ const ( // getPrebuiltImage returns the pre-built tailscale-rs Docker image name if set. func getPrebuiltImage() string { - return os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE") + return envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE") } // TailscaleRustInContainer runs the tailscale-rs axum example as an @@ -210,9 +210,9 @@ func New( if prebuiltImage := getPrebuiltImage(); prebuiltImage != "" { log.Printf("Using pre-built tailscale-rs image: %s", prebuiltImage) - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, fmt.Errorf("tsric: invalid image format %q, expected repository:tag", prebuiltImage) //nolint:err113 + repo, tag, err := integrationutil.ParseImageRef(prebuiltImage) + if err != nil { + return nil, fmt.Errorf("tsric: %w", err) } runOptions.Repository = repo diff --git a/nix/images.nix b/nix/images.nix new file mode 100644 index 000000000..1a91453cb --- /dev/null +++ b/nix/images.nix @@ -0,0 +1,236 @@ +# Nix-built + pinned container images for the integration tests. +# +# Built images (headscale, tailscale-HEAD, derper) replace +# Dockerfile.integration-ci / Dockerfile.tailscale-HEAD / Dockerfile.derper for +# the nix/flake-check path. Released tailscale versions + postgres are pinned +# pulls (nix/tailscale-versions.nix). Everything is loaded into the VM docker so +# the checks run the full version matrix offline; the Go test code is unchanged +# and consumes them via the same HEADSCALE_INTEGRATION_*_IMAGE env vars / repo +# tags the `hi` CI path already uses. +{ pkgs, buildGoModule, tailscaleSrc }: +let + lib = pkgs.lib; + + # Single source of truth for the tailscale version matrix + pinned registry + # images, shared with the Go suite (MustTestVersions reads the same file). + # Regenerate with `update-integration-images`. + versions = builtins.fromJSON (builtins.readFile ../integration/tailscale-versions.json); + + # amd64/arm64 follows the build system; the tailscale + postgres registry + # images are multi-arch so the imageDigest (the index) is shared and we select + # the per-arch nix hash. Run on whatever arch the runner is, no forced cross-build. + dockerArch = if pkgs.stdenv.hostPlatform.isAarch64 then "arm64" else "amd64"; + + # Same headscale build, but from the Go-only filtered source and a fixed + # version (not the commit rev) so the image is content-addressed: identical + # source → identical store path → shared-cache hit across commits. + headscale = pkgs.headscale.overrideAttrs (_: { + src = pkgs.headscale-go-src; + version = "integration"; + }); + + # alpine's `update-ca-certificates` has no drop-in nixpkgs equivalent. The + # integration containers write the per-test headscale CA to + # /usr/local/share/ca-certificates/user-N.crt and then call + # update-ca-certificates so TLS to headscale is trusted (integration/tsic, + # integration/hsic). This shim reproduces exactly that: append those certs + # onto a writable bundle that Go reads via SSL_CERT_FILE. + # + # ponytail: minimal CA shim; swap for a real ca-certificates package only if a + # test needs the full openssl rehash dance. + caShim = pkgs.writeShellScriptBin "update-ca-certificates" '' + set -eu + bundle=/etc/ssl/certs/ca-certificates.crt + mkdir -p /etc/ssl/certs + if [ ! -s "$bundle" ]; then + cp ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt "$bundle" + fi + for f in /usr/local/share/ca-certificates/*.crt; do + [ -e "$f" ] || continue + cat "$f" >> "$bundle" + done + ''; + + # taildrive (TestGrantCapDrive) spawns its per-user fileserver by re-execing + # tailscaled under `su` (tailscale drive/driveimpl/remote_impl.go): tailscaled + # runs as root and sus to the share's user. With no `su` on PATH canSU() fails, + # the fallback hits assertNotRoot() while root, the fileserver never starts, + # and every share fails with "unable to determine address for share". The + # released alpine images ship busybox su; expose just that applet here so it + # does not shadow coreutils on the image PATH. + busyboxSu = pkgs.runCommand "busybox-su" { } '' + mkdir -p $out/bin + ln -s ${pkgs.busybox}/bin/busybox $out/bin/su + ''; + + # Tailscale HEAD client + daemon + derper, built once from the pinned source. + # Mirrors Dockerfile.tailscale-HEAD / Dockerfile.derper's `go install`. + tailscaleHead = buildGoModule { + pname = "tailscale-head"; + version = "head"; + src = tailscaleSrc; + + # Tailscale commits its own Go module vendorHash to flakehashes.json; reuse + # it so `nix flake update tailscale-head` carries the matching hash and we + # never hand-maintain it. (The hash tracks go.mod, not our subPackages.) + vendorHash = (builtins.fromJSON (builtins.readFile "${tailscaleSrc}/flakehashes.json")).vendor.sri; + + subPackages = [ "cmd/tailscale" "cmd/tailscaled" "cmd/containerboot" "cmd/derper" ]; + + # ts_debug_websockets so this one image also serves the websocket-DERP test + # (no separate build); the code path is inert without TS_DEBUG_DERP_WS_CLIENT. + tags = [ "ts_debug_websockets" ]; + + env.CGO_ENABLED = "0"; + doCheck = false; + }; + + # Userland the integration suite shells out to inside the containers: + # sh/sleep/cat (coreutils), find (findutils), ps/pidof (procps), + # grep/sed/awk, bash. Missing any of these makes test Execute() calls fail. + baseUserland = with pkgs; [ + bashInteractive + coreutils + findutils + procps + gnugrep + gnused + gawk + ]; + + # Shared image scaffolding. Go reads /etc/ssl/certs/ca-certificates.crt via + # SSL_CERT_FILE (maintained by caShim at container start). + commonEnv = [ + "PATH=/bin:/usr/bin:/usr/local/bin:/sbin" + "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt" + ]; + # /var/lib/tailscale must exist and be writable, else tailscaled's default + # state path resolves empty and it exits with "--state is required" + # (tailscale paths/paths_unix.go stateFileUnix). derper-certs holds the + # manual cert dsic writes before starting derper. + mkWritableDirs = '' + mkdir -p etc/ssl/certs usr/local/share/ca-certificates usr/local/share/derper-certs etc/headscale tmp var/run var/lib/tailscale + ''; + + # Local login accounts the SSH integration tests map tailscale SSH sessions to. + # Pre-created in both images at build time (head: seeded passwd below; released: + # busybox adduser in withTestTools) so no container is mutated at startup. + sshTestUsers = [ "ssh-it-user" "user1" "user2" ]; + passwdFile = pkgs.writeText "passwd" ('' + root:x:0:0:root:/root:/bin/sh + nobody:x:65534:65534:nobody:/:/bin/sh + '' + lib.concatStrings (lib.imap0 + (i: u: "${u}:x:${toString (1000 + i)}:${toString (1000 + i)}::/home/${u}:/bin/sh\n") + sshTestUsers)); + groupFile = pkgs.writeText "group" ('' + root:x:0: + nobody:x:65534: + '' + lib.concatStrings (lib.imap0 (i: u: "${u}:x:${toString (1000 + i)}:\n") sshTestUsers)); + + tailscaleImage = pkgs.dockerTools.buildLayeredImage { + name = "tailscale-head"; + tag = "nix"; + contents = [ + tailscaleHead + caShim + pkgs.dockerTools.binSh # /bin/sh -> bash (tsic entrypoint is /bin/sh -c) + pkgs.iproute2 # ip + pkgs.iptables # iptables, ip6tables + pkgs.curl + pkgs.python3 # webserver tests + pkgs.dnsutils # dig/nslookup (dns tests apk-add bind-tools) + pkgs.openssh # ssh, for the SSH tests + pkgs.getent # tailscale SSH server execs `getent passwd ` + pkgs.hostname # SSH tests run `hostname` on the peer over the session + pkgs.traceroute # HA/route tests assert path via traceroute (head-only) + busyboxSu # taildrive sus to spawn its per-user fileserver (TestGrantCapDrive) + ] ++ baseUserland; + # buildLayeredImage puts every package's bin on /bin, which is on PATH, so + # the suite resolves tailscale/tailscaled/derper/curl/ssh/... directly — no + # symlinks needed. Just the writable state dirs and a passwd with the login + # the SSH tests use (ssh-it-user, pre-created so no runtime adduser). + extraCommands = '' + ${mkWritableDirs} + ${lib.concatMapStringsSep "\n" (u: "mkdir -p home/${u}") sshTestUsers} + cp ${passwdFile} etc/passwd + cp ${groupFile} etc/group + ''; + config.Env = commonEnv; + }; + + headscaleImage = pkgs.dockerTools.buildLayeredImage { + name = "headscale"; + tag = "nix"; + contents = [ + headscale + caShim + pkgs.dockerTools.binSh # hsic entrypoint is /bin/bash -c + pkgs.iproute2 # ip + pkgs.jq + pkgs.sqlite + pkgs.python3 + pkgs.curl + pkgs.dnsutils # dig + ] ++ baseUserland; + extraCommands = '' + ${mkWritableDirs} + mkdir -p usr/local/bin + ln -sf ${headscale}/bin/headscale usr/local/bin/headscale + ''; + config.Env = commonEnv; + }; + + # The tools the suite used to `apk add` into the released alpine images at + # runtime (curl/ssh/dig/python3/getent). A nixosTest has no network, so we bake + # them in at build time instead — one buildEnv copied onto the image so they + # land on /bin (already on the alpine PATH), no per-tool symlinks. Only /bin is + # linked: overlaying /lib would shadow musl's loader and break alpine's + # tailscaled. The binaries carry their own glibc closure via RPATH, so they run + # on musl without it. Tests assert these are present rather than installing them. + testTools = pkgs.buildEnv { + name = "ts-test-tools"; + paths = with pkgs; [ curl openssh dnsutils python3 getent ]; + pathsToLink = [ "/bin" ]; + }; + + # Pinned registry images, as docker-loadable tarballs that keep their + # repo:tag so the suite finds them locally (its own pull fails offline, then + # RunWithOptions uses the loaded image). The spec's sha256 is per-arch. + pullImage = spec: pkgs.dockerTools.pullImage { + imageName = spec.imageName; + finalImageName = spec.imageName; + finalImageTag = spec.finalImageTag; + imageDigest = spec.imageDigest; + os = "linux"; + arch = dockerArch; + sha256 = spec.sha256.${dockerArch}; + }; + + # Released tailscale images with the test tools layered on, keeping the exact + # registry repo:tag the suite requests, and the SSH login pre-created at build + # time (alpine's busybox adduser) so no container is mutated at startup — + # matching the head image's baked ssh-it-user. + withTestTools = spec: pkgs.dockerTools.buildImage { + name = spec.imageName; + tag = spec.finalImageTag; + fromImage = pullImage spec; + copyToRoot = testTools; + runAsRoot = '' + #!${pkgs.runtimeShell} + export PATH=/usr/sbin:/usr/bin:/sbin:/bin + ${lib.concatMapStringsSep "\n" (u: "grep -q '^${u}:' /etc/passwd || adduser -D -s /bin/sh ${u}") sshTestUsers} + ''; + }; + + tailscaleVersionImages = map withTestTools (builtins.attrValues versions.images); + postgresImage = pullImage versions.postgres; +in +{ + inherit + tailscaleImage + headscaleImage + postgresImage + tailscaleVersionImages + tailscaleHead + caShim; +} diff --git a/nix/tests/integration.nix b/nix/tests/integration.nix new file mode 100644 index 000000000..e96a861d9 --- /dev/null +++ b/nix/tests/integration.nix @@ -0,0 +1,157 @@ +# Per-test integration check: a NixOS VM that boots docker, loads the nix-built +# + pinned images (full tailscale version matrix), and runs the prebuilt +# integration test binary filtered to a single test. Hermetic equivalent of one +# `go run ./cmd/hi run ` job. `postgres = true` produces the postgres +# variant (HEADSCALE_INTEGRATION_POSTGRES=1) for the postgres subset. +# +# The suite runs *inside* a container named headscale-test-suite- with +# the docker socket mounted (docker-out-of-docker): scenario.go attaches that +# container to each test network, and IntegrationSkip gates on /.dockerenv. We +# reuse the headscale image as the runner base and bind-mount the host nix store +# so the test binary's closure resolves. +{ name +, testFilter ? name +, pkgs +, headscaleImage +, tailscaleImage +, postgresImage +, tailscaleVersionImages +, testBin +, src +, postgres ? false +}: +let + lib = pkgs.lib; + + # The test function name, dropping any "/subtest" filter on a split check, so + # the skip-guard below matches a whole-function skip without tripping on a + # legitimately-skipped subtest. + testFunc = builtins.head (lib.splitString "/" testFilter); + + # >=6 chars and DNS-safe: tsic/hsic derive container hostnames from the last + # 6 chars of the run ID (runID[len-6:]), which panics on shorter ids. + runID = "nixcheck"; + runnerName = "headscale-test-suite-${runID}"; + + # Images this check must have locally before the suite runs. Every test may + # fan out over the full tailscale version matrix (MustTestVersions); the head + # image doubles as the DERP server; the postgres variant adds postgres. + images = + [ headscaleImage tailscaleImage ] + ++ tailscaleVersionImages + ++ lib.optional postgres postgresImage; + + # Loading ~8 images dominates wall time; do it in parallel (they share layers, + # so docker dedups) and fail if any load fails. + loadImages = pkgs.writeShellScript "load-integration-images" '' + set -u + pids="" + ${lib.concatMapStringsSep "\n" (img: ''docker load -i ${img} & pids="$pids $!"'') images} + rc=0 + for p in $pids; do wait "$p" || rc=1; done + exit "$rc" + ''; + + pgEnv = lib.optionalString postgres + "-e HEADSCALE_INTEGRATION_POSTGRES=1 -e HEADSCALE_INTEGRATION_POSTGRES_IMAGE=postgres:16 "; +in +{ + name = "integration-${name}${lib.optionalString postgres "-pg"}"; + + nodes.machine = { ... }: { + virtualisation.docker.enable = true; + + # Match the integration CI: Docker 29's default containerd snapshotter + + # overlayfs regresses image load/exec for these images; the classic + # overlay2 graph driver is the known-good path. + virtualisation.docker.daemon.settings = { + storage-driver = "overlay2"; + features.containerd-snapshotter = false; + }; + + # tailscaled needs /dev/net/tun for its --tun=tsdev device. + boot.kernelModules = [ "tun" ]; + + # The VM's dhcpcd otherwise manages docker's bridges/veths: as containers + # and the cable-pull tests churn interfaces it deletes their routes and + # races libnetwork's gateway setup, surfacing as + # "API error (500): updating gateway endpoint: failed to set gateway: + # network is unreachable" on reconnect (TestHASubnetRouterFailover*Disconnect + # /CablePull) and intermittent container-startup flakes. Keep it off them. + networking.dhcpcd.denyInterfaces = [ "veth*" "br-*" "docker*" ]; + + # Trim the guest toward a tiny profile: this VM only needs to boot, run + # docker, and exec one test binary. Measured boot is 18.7s (1.7s kernel + + # 3.9s initrd + 13.1s userspace); the userspace critical chain is + # dhcpcd (6.1s) -> network-online.target -> docker.service (+3.5s), i.e. the + # remaining cost is functional (docker waits for the network), not fat to + # trim. documentation + timesyncd off shave the cheap, avoidable part. + # + # microvm.nix was evaluated against this and rejected: it still needs a full + # docker daemon + nested KVM, and boot is only ~10-15% of the 2-3 min + # image-load + test runtime, so a new flake input can't pay itself back. The + # one real lever left, dropping docker's network-online wait (~6s), sits + # right next to the dhcpcd interface races handled below and isn't worth + # destabilising the matrix for. + documentation.enable = false; + services.timesyncd.enable = false; + + # Sized for the full version matrix (a test can spawn 12+ tailscale + # containers + headscale + the runner) while still fitting a standard + # 4-core/16 GB GitHub runner. Disk-backed docker (not tmpfs) keeps RAM + # headroom on the runner; the slim source copy below is the bigger win. + virtualisation.memorySize = 8192; + virtualisation.cores = 4; + virtualisation.diskSize = 12288; + }; + + testScript = '' + start_all() + machine.wait_for_unit("docker.service") + + # Load all matrix images in parallel (store paths visible inside the VM). + machine.succeed("${loadImages}", timeout=600) + + # Writable copy of just the integration package (the suite writes + # control_logs/ under CWD and reads testdata relative to it). Only this + # subtree is needed at runtime — the repo-root build context is only used + # for image builds, which we bypass with prebuilt images. + machine.succeed("mkdir -p /tmp/hs && cp -r ${src}/integration /tmp/hs/integration && chmod -R u+w /tmp/hs") + + # hsic/tsic save artifacts (and tsic.Status writes _status.json) under + # /tmp/control; cmd/hi mounts this dir. Without it every Status() errors on + # the write and the suite never observes NeedsLogin. + machine.succeed("mkdir -p /tmp/control") + + # Run the suite inside a container (docker-out-of-docker), mirroring cmd/hi: + # - name matches scenario.go's testSuiteName so it can join test networks + # - docker socket mounted so sibling containers spawn on this daemon + # - host nix store mounted so the test binary's closure resolves + out = machine.succeed( + "docker run --rm --name ${runnerName} " + "-v /var/run/docker.sock:/var/run/docker.sock " + "-v /nix/store:/nix/store:ro " + "-v /tmp/hs:/tmp/hs -w /tmp/hs/integration " + "-v /tmp/control:/tmp/control " + "-e HEADSCALE_INTEGRATION_HEADSCALE_IMAGE=headscale:nix " + "-e HEADSCALE_INTEGRATION_TAILSCALE_IMAGE=tailscale-head:nix " + "-e HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE=tailscale-head:nix " + "-e HEADSCALE_INTEGRATION_DERPER_IMAGE=tailscale-head:nix " + "${pgEnv}" + "-e HEADSCALE_INTEGRATION_RUN_ID=${runID} " + "-e CI=1 " + # The nested-docker VM converges slower than a host docker daemon; widen + # the ScaledTimeout budgets (subnet-route convergence, HA cycles) past 2x. + "-e HEADSCALE_INTEGRATION_TIMEOUT_SCALE=4 " + "-e SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt " + "headscale:nix " + "${testBin}/bin/integration.test -test.run '^${testFilter}$' -test.v -test.timeout=20m", + timeout=1800, + ) + print(out) + + # A skipped or unmatched test still exits 0; guard against a false green. + assert "no tests to run" not in out, "filter ${testFilter} matched nothing" + assert "--- SKIP: ${testFunc} " not in out, "test ${testFunc} was skipped (not in container?)" + ''; +} diff --git a/tools/tailscale-versions/main.go b/tools/tailscale-versions/main.go new file mode 100644 index 000000000..814bd6bd0 --- /dev/null +++ b/tools/tailscale-versions/main.go @@ -0,0 +1,217 @@ +// Command tailscale-versions regenerates integration/tailscale-versions.json: +// the bare version list the suite tests (from capver) plus the per-arch +// registry pins (manifest digest + nix sha256) the offline nix checks load. +// +// It is wired as a //go:generate step in the integration package so a capver +// bump refreshes the pins in the same `make generate` pass as capver itself. It +// shells out to nix-prefetch-docker, so it runs in the nix dev shell — but only +// for tags that aren't already pinned: a routine generate with an unchanged +// version set makes no network call and rewrites nothing. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + + "github.com/juanfont/headscale/hscontrol/capver" +) + +// versionsJSONPath is relative to the integration package directory, which is +// the working directory go:generate runs this command from. +const versionsJSONPath = "tailscale-versions.json" + +// genArches are the docker architectures pinned for every image. The registry +// images are multi-arch, so the imageDigest (the manifest index) is shared and +// only the nix sha256 differs per arch; pullImage selects by the build system. +var genArches = []string{"amd64", "arm64"} + +var ( + reImageDigest = regexp.MustCompile(`imageDigest = "([^"]+)"`) + reHash = regexp.MustCompile(`hash = "([^"]+)"`) + + errPrefetchEmpty = errors.New("nix-prefetch-docker produced no digest/hash (rate limited? retry)") +) + +// imageRef pins a multi-arch registry image for dockerTools.pullImage. +type imageRef struct { + ImageName string `json:"imageName"` + FinalImageTag string `json:"finalImageTag"` + ImageDigest string `json:"imageDigest"` + SHA256 map[string]string `json:"sha256"` +} + +// complete reports whether the pin already has a digest and every arch hash, so +// it can be reused without re-hitting the registry. +func (r imageRef) complete() bool { + if r.ImageDigest == "" { + return false + } + + for _, a := range genArches { + if r.SHA256[a] == "" { + return false + } + } + + return true +} + +// versionsFile is the single source of truth shared by the Go suite +// (MustTestVersions reads .versions) and the nix image pins (images.nix reads +// .images / .postgres). See integration/tailscale-versions.json. +type versionsFile struct { + Versions []string `json:"versions"` + Images map[string]imageRef `json:"images"` + Postgres imageRef `json:"postgres"` +} + +func main() { + err := run() + if err != nil { + fmt.Fprintln(os.Stderr, "tailscale-versions:", err) + os.Exit(1) + } +} + +func run() error { + _, must := capver.IntegrationVersions() + + prev := readExisting() + + out := versionsFile{Versions: must, Images: map[string]imageRef{}} + + for _, v := range must { + // head is built from source, not pulled. + if v == "head" { + continue + } + + repo, tag := "ghcr.io/tailscale/tailscale", "v"+v + if v == "unstable" { + repo, tag = "tailscale/tailscale", "unstable" + } + + ref, err := pinOrReuse(prev.Images[tag], repo, tag) + if err != nil { + return err + } + + out.Images[tag] = ref + } + + postgres, err := pinOrReuse(prev.Postgres, "postgres", "16") + if err != nil { + return err + } + + out.Postgres = postgres + + return writeIfChanged(out) +} + +// pinOrReuse keeps an already-complete pin so a routine generate makes no +// network call; only new or incomplete tags are prefetched. To deliberately +// refresh a mutable tag (unstable, postgres) delete its entry first. +func pinOrReuse(prev imageRef, repo, tag string) (imageRef, error) { + if prev.complete() { + return prev, nil + } + + return pinImage(context.Background(), repo, tag) +} + +// readExisting loads the current pins, or an empty file if absent/unparseable +// (a full regeneration). +func readExisting() versionsFile { + vf := versionsFile{Images: map[string]imageRef{}} + + b, err := os.ReadFile(versionsJSONPath) + if err != nil { + return vf + } + + err = json.Unmarshal(b, &vf) + if err != nil { + return versionsFile{Images: map[string]imageRef{}} + } + + if vf.Images == nil { + vf.Images = map[string]imageRef{} + } + + return vf +} + +// writeIfChanged only touches the file when the rendered JSON differs, so a +// no-op generate leaves the working tree (and git) clean. +func writeIfChanged(out versionsFile) error { + b, err := json.MarshalIndent(out, "", " ") + if err != nil { + return fmt.Errorf("marshalling versions: %w", err) + } + + b = append(b, '\n') + + old, err := os.ReadFile(versionsJSONPath) + if err == nil && bytes.Equal(old, b) { + fmt.Printf("%s already up to date\n", versionsJSONPath) + + return nil + } + + err = os.WriteFile(versionsJSONPath, b, 0o600) + if err != nil { + return fmt.Errorf("writing %s: %w", versionsJSONPath, err) + } + + fmt.Printf("wrote %s\n", versionsJSONPath) + + return nil +} + +// pinImage prefetches repo:tag for every arch via nix-prefetch-docker and +// returns the shared manifest digest plus per-arch nix hashes. +func pinImage(ctx context.Context, repo, tag string) (imageRef, error) { + ref := imageRef{ImageName: repo, FinalImageTag: tag, SHA256: map[string]string{}} + + for _, arch := range genArches { + cmd := exec.CommandContext(ctx, "nix", "run", "nixpkgs#nix-prefetch-docker", "--", + "--image-name", repo, "--image-tag", tag, "--arch", arch, "--os", "linux") + + var stdout bytes.Buffer + + cmd.Stdout = &stdout + + err := cmd.Run() + if err != nil { + return imageRef{}, fmt.Errorf("prefetch %s:%s (%s): %w", repo, tag, arch, err) + } + + digest := firstSubmatch(reImageDigest, stdout.String()) + hash := firstSubmatch(reHash, stdout.String()) + + if digest == "" || hash == "" { + return imageRef{}, fmt.Errorf("%s:%s (%s): %w", repo, tag, arch, errPrefetchEmpty) + } + + ref.ImageDigest = digest + ref.SHA256[arch] = hash + } + + return ref, nil +} + +func firstSubmatch(re *regexp.Regexp, s string) string { + m := re.FindStringSubmatch(s) + if len(m) < 2 { + return "" + } + + return m[1] +}