integration: support prebuilt images and the slow nix VM in the suite

Read the HEADSCALE_INTEGRATION_*_IMAGE knobs through one envknob helper and
assert required tooling is present instead of installing it at runtime (the
nix VM is offline). dsic waits for both the derper cert and key before starting
derper, so the container can't die between the two uploads. The nested-docker
VM is slower than a host daemon, so every integration timeout (convergence,
docker exec, peer sync, pool wait) now flows through one
dockertestutil.ScaleTimeout honouring HEADSCALE_INTEGRATION_TIMEOUT_SCALE.

integration/dockertestutil/execute.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"os"
+	"strconv"
 	"sync"
 	"time"
 
@@ -11,15 +13,31 @@ import (
 	"github.com/ory/dockertest/v3"
 )
 
-// defaultExecuteTimeout returns the timeout for docker exec commands.
-// On CI runners, docker exec latency is higher due to resource
-// contention, so the timeout is doubled.
-func defaultExecuteTimeout() time.Duration {
-	if util.IsCI() {
-		return 20 * time.Second
+// ScaleTimeout scales an integration timeout for the running environment: the
+// nested-docker nix-VM checks are slower than a host docker daemon and set
+// HEADSCALE_INTEGRATION_TIMEOUT_SCALE to widen every budget; otherwise CI uses
+// 2x and local dev the base. One knob covers exec, convergence, peer sync, and
+// pool waits. Lives here (the lowest integration package) so every layer can
+// reach it.
+func ScaleTimeout(base time.Duration) time.Duration {
+	scale := os.Getenv("HEADSCALE_INTEGRATION_TIMEOUT_SCALE")
+	if scale != "" {
+		f, err := strconv.ParseFloat(scale, 64)
+		if err == nil && f > 0 {
+			return time.Duration(float64(base) * f)
+		}
 	}
 
-	return 10 * time.Second
+	if util.IsCI() {
+		return base * 2
+	}
+
+	return base
+}
+
+// defaultExecuteTimeout returns the timeout for docker exec commands.
+func defaultExecuteTimeout() time.Duration {
+	return ScaleTimeout(10 * time.Second)
 }
 
 var (

integration/dsic/dsic.go

@@ -117,9 +117,14 @@ func (dsic *DERPServerInContainer) buildEntrypoint(derperArgs string) []string {
 	// Wait for network to be ready
 	commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done")
 
-	// Wait for TLS cert to be written (always written after container start)
+	// Wait for the TLS cert AND key to be written (both written after container
+	// start). Waiting only for the cert races: once it lands derper starts, and
+	// since it also needs the key — written a moment later — it exits and kills
+	// the container before the key upload (reliably, with slow nested-docker exec).
 	commands = append(commands,
 		fmt.Sprintf("while [ ! -f %s/%s.crt ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname))
+	commands = append(commands,
+		fmt.Sprintf("while [ ! -f %s/%s.key ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname))
 
 	// If CA certs are configured, wait for them to be written
 	if len(dsic.caCerts) > 0 {
@@ -221,34 +226,56 @@ func New(
 
 	var container *dockertest.Resource
 
-	buildOptions := &dockertest.BuildOptions{
-		Dockerfile: "Dockerfile.derper",
-		ContextDir: dockerContextPath,
-		BuildArgs:  []docker.BuildArg{},
-	}
-
-	switch version {
-	case "head":
-		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
-			Name:  "VERSION_BRANCH",
-			Value: "main",
-		})
-	default:
-		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
-			Name:  "VERSION_BRANCH",
-			Value: "v" + version,
-		})
-	}
 	// Add integration test labels if running under hi tool
 	dockertestutil.DockerAddIntegrationLabels(runOptions, "derp")
 
-	container, err = pool.BuildAndRunWithBuildOptions(
-		buildOptions,
-		runOptions,
-		dockertestutil.DockerRestartPolicy,
-		dockertestutil.DockerAllowLocalIPv6,
-		dockertestutil.DockerAllowNetworkAdministration,
-	)
+	// In CI / nix checks a prebuilt derper image is provided so we neither build
+	// the image nor clone tailscale at runtime, mirroring the tailscale and
+	// headscale image injection. Only the head image is prebuilt.
+	repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_DERPER_IMAGE")
+	if err != nil {
+		return nil, err
+	}
+
+	if prebuilt && version == "head" {
+		runOptions.Repository = repo
+		runOptions.Tag = tag
+
+		container, err = pool.RunWithOptions(
+			runOptions,
+			dockertestutil.DockerRestartPolicy,
+			dockertestutil.DockerAllowLocalIPv6,
+			dockertestutil.DockerAllowNetworkAdministration,
+		)
+	} else {
+		buildOptions := &dockertest.BuildOptions{
+			Dockerfile: "Dockerfile.derper",
+			ContextDir: dockerContextPath,
+			BuildArgs:  []docker.BuildArg{},
+		}
+
+		switch version {
+		case "head":
+			buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
+				Name:  "VERSION_BRANCH",
+				Value: "main",
+			})
+		default:
+			buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
+				Name:  "VERSION_BRANCH",
+				Value: "v" + version,
+			})
+		}
+
+		container, err = pool.BuildAndRunWithBuildOptions(
+			buildOptions,
+			runOptions,
+			dockertestutil.DockerRestartPolicy,
+			dockertestutil.DockerAllowLocalIPv6,
+			dockertestutil.DockerAllowNetworkAdministration,
+		)
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf(
 			"%s starting tailscale DERPer container (version: %s): %w",

integration/general_test.go

@@ -483,16 +483,7 @@ func TestTaildrop(t *testing.T) {
 	_, err = scenario.ListTailscaleClientsFQDNs()
 	requireNoErrListFQDN(t, err)
 
-	// Install curl on all clients
-	for _, client := range allClients {
-		if !strings.Contains(client.Hostname(), "head") {
-			command := []string{"apk", "add", "curl"}
-			_, _, err := client.Execute(command)
-			if err != nil {
-				t.Fatalf("failed to install curl on %s, err: %s", client.Hostname(), err)
-			}
-		}
-	}
+	// curl is baked into every image (no runtime install).
 
 	// Helper to get FileTargets for a client.
 	getFileTargets := func(client TailscaleClient) ([]apitype.FileTarget, error) {

integration/helpers.go

@@ -22,6 +22,7 @@ import (
 	policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/util"
+	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/oauth2-proxy/mockoidc"
@@ -922,13 +923,7 @@ func assertCommandOutputContains(t *testing.T, c TailscaleClient, command []stri
 // Uses longer timeouts in CI environments to account for slower resource allocation
 // and higher system load during automated testing.
 func dockertestMaxWait() time.Duration {
-	wait := 300 * time.Second //nolint
-
-	if util.IsCI() {
-		wait = 600 * time.Second //nolint
-	}
-
-	return wait
+	return dockertestutil.ScaleTimeout(300 * time.Second)
 }
 
 // didClientUseWebsocketForDERP analyzes client logs to determine if WebSocket was used for DERP.

integration/hsic/hsic.go

@@ -57,10 +57,8 @@ const (
 )
 
 var (
-	errHeadscaleStatusCodeNotOk    = errors.New("headscale status code not ok")
-	errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag")
-	errHeadscaleImageRequiredInCI  = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI")
-	errInvalidPostgresImageFormat  = errors.New("invalid HEADSCALE_INTEGRATION_POSTGRES_IMAGE format, expected repository:tag")
+	errHeadscaleStatusCodeNotOk   = errors.New("headscale status code not ok")
+	errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI")
 )
 
 type fileInContainer struct {
@@ -406,12 +404,12 @@ func New(
 		pgRepo := "postgres"
 		pgTag := "latest"
 
-		if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_POSTGRES_IMAGE"); prebuiltImage != "" {
-			repo, tag, found := strings.Cut(prebuiltImage, ":")
-			if !found {
-				return nil, errInvalidPostgresImageFormat
-			}
+		repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_POSTGRES_IMAGE")
+		if err != nil {
+			return nil, err
+		}
 
+		if prebuilt {
 			pgRepo = repo
 			pgTag = tag
 		}
@@ -510,16 +508,13 @@ func New(
 	var container *dockertest.Resource
 
 	// Check if a pre-built image is available via environment variable
-	prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE")
-
-	if prebuiltImage != "" {
-		log.Printf("Using pre-built headscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value
-		// Parse image into repository and tag
-		repo, tag, ok := strings.Cut(prebuiltImage, ":")
-		if !ok {
-			return nil, errInvalidHeadscaleImageFormat
-		}
+	repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE")
+	if err != nil {
+		return nil, err
+	}
 
+	if prebuilt {
+		log.Printf("Using pre-built headscale image: %s:%s", repo, tag) //nolint:gosec // G706: integration-only log of trusted env value
 		runOptions.Repository = repo
 		runOptions.Tag = tag
 
@@ -530,7 +525,7 @@ func New(
 			dockertestutil.DockerAllowNetworkAdministration,
 		)
 		if err != nil {
-			return nil, fmt.Errorf("running pre-built headscale container %q: %w", prebuiltImage, err)
+			return nil, fmt.Errorf("running pre-built headscale container %s:%s: %w", repo, tag, err)
 		}
 	} else if util.IsCI() {
 		return nil, errHeadscaleImageRequiredInCI

integration/integrationutil/util.go

@@ -8,28 +8,61 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"io"
 	"math/big"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/juanfont/headscale/hscontrol/types"
-	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
+	"tailscale.com/envknob"
 	"tailscale.com/tailcfg"
 )
 
-// PeerSyncTimeout returns the timeout for peer synchronization based on environment:
-// 60s for dev, 120s for CI.
-func PeerSyncTimeout() time.Duration {
-	if util.IsCI() {
-		return 120 * time.Second
+var errInvalidImageFormat = errors.New("integration image env must be in repository:tag format")
+
+// PrebuiltImage reads a HEADSCALE_INTEGRATION_*_IMAGE knob (the full var name,
+// e.g. "HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") and splits it into repository
+// and tag. The bool is false when the knob is unset — the suite then builds the
+// image itself; it errors when the value is set but is not "repository:tag".
+// This is the one place the prebuilt-image knobs (used by the CI / nix-check
+// path) are read, via tailscale's envknob.
+func PrebuiltImage(envVar string) (string, string, bool, error) {
+	image := envknob.String(envVar)
+	if image == "" {
+		return "", "", false, nil
 	}
 
-	return 60 * time.Second
+	repo, tag, err := ParseImageRef(image)
+	if err != nil {
+		return "", "", false, fmt.Errorf("%s=%w", envVar, err)
+	}
+
+	return repo, tag, true, nil
+}
+
+// ParseImageRef splits an already-resolved "repository:tag" image string into
+// its parts. Use it where the image comes from somewhere other than a single
+// knob (e.g. a websocket-tagged override chosen at runtime); for the common
+// read-a-knob case use [PrebuiltImage].
+func ParseImageRef(image string) (string, string, error) {
+	repo, tag, found := strings.Cut(image, ":")
+	if !found {
+		return "", "", fmt.Errorf("%q: %w", image, errInvalidImageFormat)
+	}
+
+	return repo, tag, nil
+}
+
+// PeerSyncTimeout returns the timeout for peer synchronization: 60s for dev,
+// scaled for CI / the slow nix VM.
+func PeerSyncTimeout() time.Duration {
+	return dockertestutil.ScaleTimeout(60 * time.Second)
 }
 
 // PeerSyncRetryInterval returns the retry interval for peer synchronization checks.
@@ -37,16 +70,10 @@ func PeerSyncRetryInterval() time.Duration {
 	return 100 * time.Millisecond
 }
 
-// ScaledTimeout returns the given timeout, scaled for CI environments
-// where resource contention causes slower state propagation.
-// Uses a 2x multiplier, consistent with PeerSyncTimeout (60s/120s)
-// and dockertestMaxWait (300s/600s).
+// ScaledTimeout returns the given convergence budget, scaled for the running
+// environment via [dockertestutil.ScaleTimeout].
 func ScaledTimeout(d time.Duration) time.Duration {
-	if util.IsCI() {
-		return d * 2
-	}
-
-	return d
+	return dockertestutil.ScaleTimeout(d)
 }
 
 func WriteFileToContainer(

integration/scenario.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"context"
 	"crypto/tls"
+	_ "embed"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -22,7 +23,6 @@ import (
 	"time"
 
 	clientv1 "github.com/juanfont/headscale/gen/client/v1"
-	"github.com/juanfont/headscale/hscontrol/capver"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/dsic"
@@ -54,30 +54,38 @@ var (
 	errNoHeadscaleAvailable = errors.New("no headscale available")
 	errNoUserAvailable      = errors.New("no user available")
 	errNoClientFound        = errors.New("client not found")
-
-	// AllVersions represents a list of Tailscale versions the suite
-	// uses to test compatibility with the [ControlServer].
-	//
-	// The list contains two special cases, "head" and "unstable" which
-	// points to the current tip of Tailscale's main branch and the latest
-	// released unstable version.
-	//
-	// The rest of the version represents Tailscale versions that can be
-	// found in Tailscale's apt repository.
-	AllVersions = append([]string{"head", "unstable"}, capver.TailscaleLatestMajorMinor(capver.SupportedMajorMinorVersions, true)...)
-
-	// MustTestVersions is the minimum set of versions we should test.
-	// At the moment, this is arbitrarily chosen as:
-	//
-	// - Two unstable (HEAD and unstable)
-	// - Two latest versions
-	// - Two oldest supported version.
-	MustTestVersions = append(
-		AllVersions[0:4],
-		AllVersions[len(AllVersions)-2:]...,
-	)
 )
 
+// Regenerate tailscale-versions.json from capver + nix-prefetch-docker. Runs in
+// the same `make generate` / `go generate ./...` pass as capver (and needs the
+// nix dev shell); already-pinned tags are reused, so an unchanged set is a
+// no-op. Refresh just the pins with `go generate ./integration/...`.
+//go:generate go run ../tools/tailscale-versions
+
+//go:embed tailscale-versions.json
+var tailscaleVersionsJSON []byte
+
+// MustTestVersions is the set of Tailscale versions the suite tests for
+// compatibility with the [ControlServer]. It contains "head" (built from
+// source) and "unstable", plus the latest and oldest supported releases. It is
+// read from the generated tailscale-versions.json — the single source shared
+// with the nix image pins (regenerate with `update-integration-images`, which
+// derives the list from capver).
+var MustTestVersions = mustTailscaleVersions()
+
+func mustTailscaleVersions() []string {
+	var v struct {
+		Versions []string `json:"versions"`
+	}
+
+	err := json.Unmarshal(tailscaleVersionsJSON, &v)
+	if err != nil {
+		panic("parsing tailscale-versions.json: " + err.Error())
+	}
+
+	return v.Versions
+}
+
 // User represents a User in the [ControlServer] and a map of [TailscaleClient]'s
 // associated with the User.
 type User struct {
@@ -1583,6 +1591,33 @@ const (
 
 var errStatusCodeNotOK = errors.New("status code not OK")
 
+// runHeadscaleImageContainer starts runOptions from the prebuilt headscale
+// image when HEADSCALE_INTEGRATION_HEADSCALE_IMAGE is set, otherwise it builds
+// Dockerfile.integration. The mock OIDC provider (headscale mockoidc) and the
+// webservice (python3 -m http.server) both run on the headscale image, so they
+// share the same prebuilt-or-build path hsic uses for headscale itself — which
+// is what lets them come up offline (e.g. in the nix VM checks).
+func (s *Scenario) runHeadscaleImageContainer(runOptions *dockertest.RunOptions) (*dockertest.Resource, error) {
+	repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE")
+	if err != nil {
+		return nil, err
+	}
+
+	if prebuilt {
+		runOptions.Repository = repo
+		runOptions.Tag = tag
+
+		return s.pool.RunWithOptions(runOptions, dockertestutil.DockerRestartPolicy)
+	}
+
+	buildOptions := &dockertest.BuildOptions{
+		Dockerfile: hsic.IntegrationTestDockerFileName,
+		ContextDir: dockerContextPath,
+	}
+
+	return s.pool.BuildAndRunWithBuildOptions(buildOptions, runOptions, dockertestutil.DockerRestartPolicy)
+}
+
 func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUser) error {
 	port, err := dockertestutil.RandomFreeHostPort()
 	if err != nil {
@@ -1618,11 +1653,6 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse
 		},
 	}
 
-	headscaleBuildOptions := &dockertest.BuildOptions{
-		Dockerfile: hsic.IntegrationTestDockerFileName,
-		ContextDir: dockerContextPath,
-	}
-
 	err = s.pool.RemoveContainerByName(hostname)
 	if err != nil {
 		return err
@@ -1633,11 +1663,7 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse
 	// Add integration test labels if running under hi tool
 	dockertestutil.DockerAddIntegrationLabels(mockOidcOptions, "oidc")
 
-	if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( //nolint:noinlineerr
-		headscaleBuildOptions,
-		mockOidcOptions,
-		dockertestutil.DockerRestartPolicy,
-	); err == nil {
+	if pmockoidc, err := s.runHeadscaleImageContainer(mockOidcOptions); err == nil { //nolint:noinlineerr
 		s.mockOIDC.r = pmockoidc
 	} else {
 		return err
@@ -1722,16 +1748,7 @@ func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) {
 	// Add integration test labels if running under hi tool
 	dockertestutil.DockerAddIntegrationLabels(webOpts, "web")
 
-	webBOpts := &dockertest.BuildOptions{
-		Dockerfile: hsic.IntegrationTestDockerFileName,
-		ContextDir: dockerContextPath,
-	}
-
-	web, err := s.pool.BuildAndRunWithBuildOptions(
-		webBOpts,
-		webOpts,
-		dockertestutil.DockerRestartPolicy,
-	)
+	web, err := s.runHeadscaleImageContainer(webOpts)
 	if err != nil {
 		return nil, err
 	}

integration/ssh_test.go

@@ -47,7 +47,6 @@ func sshScenario(t *testing.T, policy *policyv2.Policy, testName string, clients
 			// to not configure DNS.
 			tsic.WithNetfilter("off"),
 			tsic.WithPackages("openssh"),
-			tsic.WithExtraCommands("adduser ssh-it-user"),
 			tsic.WithDockerWorkdir("/"),
 		},
 		hsic.WithACLPolicy(policy),
@@ -445,7 +444,7 @@ func doSSHWithRetryAsUser(
 	peerFQDN, _ := peer.FQDN()
 
 	command := []string{
-		"/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1",
+		"ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1",
 		fmt.Sprintf("%s@%s", sshUser, peerFQDN),
 		"'hostname'",
 	}
@@ -661,7 +660,7 @@ func doSSHCheckWithTimeout(
 	peerFQDN, _ := peer.FQDN()
 
 	command := []string{
-		"/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30",
+		"ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30",
 		fmt.Sprintf("%s@%s", "ssh-it-user", peerFQDN),
 		"'hostname'",
 	}
@@ -935,7 +934,6 @@ func TestSSHOneUserToOneCheckModeOIDC(t *testing.T) {
 			tsic.WithSSH(),
 			tsic.WithNetfilter("off"),
 			tsic.WithPackages("openssh"),
-			tsic.WithExtraCommands("adduser ssh-it-user"),
 			tsic.WithDockerWorkdir("/"),
 		},
 		hsic.WithACLPolicy(sshCheckPolicy()),
@@ -1035,7 +1033,6 @@ func TestSSHCheckModeUnapprovedTimeout(t *testing.T) {
 			tsic.WithSSH(),
 			tsic.WithNetfilter("off"),
 			tsic.WithPackages("openssh"),
-			tsic.WithExtraCommands("adduser ssh-it-user"),
 			tsic.WithDockerWorkdir("/"),
 		},
 		hsic.WithACLPolicy(sshCheckPolicy()),
@@ -1607,7 +1604,6 @@ func TestSSHLocalpart(t *testing.T) {
 					tsic.WithSSH(),
 					tsic.WithNetfilter("off"),
 					tsic.WithPackages("openssh"),
-					tsic.WithExtraCommands("adduser user1", "adduser user2"),
 					tsic.WithDockerWorkdir("/"),
 				},
 				hsic.WithTestName("sshlocalpart"),

integration/tsic/tsic.go

@@ -26,6 +26,7 @@ import (
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/store/mem"
@@ -65,7 +66,6 @@ var (
 	errTailscaleWrongPeerCount         = errors.New("wrong peer count")
 	errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey")
 	errInvalidClientConfig             = errors.New("verifiably invalid client config requested")
-	errInvalidTailscaleImageFormat     = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag")
 	errTailscaleImageRequiredInCI      = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version")
 	errContainerNotInitialized         = errors.New("container not initialized")
 	errFQDNNotYetAvailable             = errors.New("FQDN not yet available")
@@ -244,20 +244,33 @@ func WithAcceptRoutes() Option {
 	}
 }
 
-// WithPackages specifies Alpine packages to install when the container starts.
-// This requires internet access and uses `apk add`. Common packages:
+// WithPackages declares the tools a test needs inside the container. The images
+// bake these in (no runtime install), so this asserts they are present at
+// startup and fails the container loudly if not. Common packages:
 // - "python3" for HTTP server
 // - "curl" for HTTP client
 // - "bind-tools" for dig command
-// - "iptables", "ip6tables" for firewall rules
-// Note: Tests using this option require internet access and cannot use
-// the built-in DERP server in offline mode.
+// - "openssh" for the ssh client.
 func WithPackages(packages ...string) Option {
 	return func(tsic *TailscaleInContainer) {
 		tsic.withPackages = append(tsic.withPackages, packages...)
 	}
 }
 
+// packageBinary maps an alpine package name to a representative binary the
+// images must provide, so [WithPackages] can assert the binary is on PATH
+// rather than installing the package at runtime.
+func packageBinary(pkg string) string {
+	switch pkg {
+	case "openssh":
+		return "ssh"
+	case "bind-tools":
+		return "dig"
+	default:
+		return pkg
+	}
+}
+
 // WithWebserver starts a Python HTTP server on the specified port
 // alongside tailscaled. This is useful for testing subnet routing
 // and ACL connectivity. Automatically adds "python3" to packages if needed.
@@ -285,20 +298,31 @@ func (t *TailscaleInContainer) buildEntrypoint() []string {
 	commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done")
 
 	// If CA certs are configured, wait for them to be written by the Go code
-	// (certs are written after container start via [TailscaleInContainer.WriteFile])
-	if len(t.caCerts) > 0 {
+	// (certs are written after container start via [TailscaleInContainer.WriteFile]).
+	// Wait for the LAST cert (highest index): they are written in order, so once
+	// user-{N-1}.crt exists every earlier one does too. Waiting only for user-0
+	// races update-ca-certificates ahead of later certs (e.g. the headscale CA at
+	// user-1 when a DERP cert occupies user-0), permanently skipping their trust.
+	if n := len(t.caCerts); n > 0 {
 		commands = append(commands,
-			fmt.Sprintf("while [ ! -f %s/user-0.crt ]; do sleep 0.1; done", caCertRoot))
+			fmt.Sprintf("while [ ! -f %s/user-%d.crt ]; do sleep 0.1; done", caCertRoot, n-1))
 	}
 
-	// Install packages if requested (requires internet access)
+	// The images bake in every tool the suite needs (nix/images.nix for the head
+	// and version images, the alpine base otherwise), so assert the requested
+	// packages are present rather than installing them — a hermetic/offline run
+	// has no package mirror. Fail the container loudly if one is missing so an
+	// image regression is obvious in its log.
 	packages := t.withPackages
 	if t.withWebserverPort > 0 && !slices.Contains(packages, "python3") {
 		packages = append(packages, "python3")
 	}
 
-	if len(packages) > 0 {
-		commands = append(commands, "apk add --no-cache "+strings.Join(packages, " "))
+	for _, pkg := range packages {
+		bin := packageBinary(pkg)
+		commands = append(commands, fmt.Sprintf(
+			"command -v %s >/dev/null 2>&1 || { echo 'tsic: required tool %s (package %s) missing from image' >&2; exit 1; }",
+			bin, bin, pkg))
 	}
 
 	// Update CA certificates
@@ -427,24 +451,31 @@ func New(
 	switch version {
 	case VersionHead:
 		// Check if a pre-built image is available via environment variable
-		prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE")
+		prebuiltImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE")
 
-		// If custom build tags are required (e.g., for websocket DERP), we cannot use
-		// the pre-built image as it won't have the necessary code compiled in.
+		// If custom build tags are required (e.g., for websocket DERP), the
+		// default pre-built image won't have the necessary code compiled in. A
+		// websocket-tagged prebuilt image can be injected separately (CI / nix
+		// checks build one with ts_debug_websockets); otherwise fall back to
+		// building the image with the tags.
 		hasBuildTags := len(tsic.buildConfig.tags) > 0
 		if hasBuildTags && prebuiltImage != "" {
-			log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value
-				prebuiltImage, tsic.buildConfig.tags)
-			prebuiltImage = ""
+			wsImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE")
+			if tsic.withWebsocketDERP && wsImage != "" {
+				prebuiltImage = wsImage
+			} else {
+				log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value
+					prebuiltImage, tsic.buildConfig.tags)
+				prebuiltImage = ""
+			}
 		}
 
 		if prebuiltImage != "" {
 			log.Printf("Using pre-built tailscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value
 
-			// Parse image into repository and tag
-			repo, tag, ok := strings.Cut(prebuiltImage, ":")
-			if !ok {
-				return nil, errInvalidTailscaleImageFormat
+			repo, tag, err := integrationutil.ParseImageRef(prebuiltImage)
+			if err != nil {
+				return nil, err
 			}
 
 			tailscaleOptions.Repository = repo

integration/tsric/tsric.go

@@ -12,13 +12,13 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"os"
 	"strings"
 
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
+	"tailscale.com/envknob"
 	"tailscale.com/util/rands"
 )
 
@@ -35,7 +35,7 @@ const (
 
 // getPrebuiltImage returns the pre-built tailscale-rs Docker image name if set.
 func getPrebuiltImage() string {
-	return os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE")
+	return envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE")
 }
 
 // TailscaleRustInContainer runs the tailscale-rs axum example as an
@@ -210,9 +210,9 @@ func New(
 	if prebuiltImage := getPrebuiltImage(); prebuiltImage != "" {
 		log.Printf("Using pre-built tailscale-rs image: %s", prebuiltImage)
 
-		repo, tag, ok := strings.Cut(prebuiltImage, ":")
-		if !ok {
-			return nil, fmt.Errorf("tsric: invalid image format %q, expected repository:tag", prebuiltImage) //nolint:err113
+		repo, tag, err := integrationutil.ParseImageRef(prebuiltImage)
+		if err != nil {
+			return nil, fmt.Errorf("tsric: %w", err)
 		}
 
 		runOptions.Repository = repo