diff --git a/integration/dockertestutil/execute.go b/integration/dockertestutil/execute.go index 79511e2e0..d49ba6455 100644 --- a/integration/dockertestutil/execute.go +++ b/integration/dockertestutil/execute.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "fmt" + "os" + "strconv" "sync" "time" @@ -11,15 +13,31 @@ import ( "github.com/ory/dockertest/v3" ) -// defaultExecuteTimeout returns the timeout for docker exec commands. -// On CI runners, docker exec latency is higher due to resource -// contention, so the timeout is doubled. -func defaultExecuteTimeout() time.Duration { - if util.IsCI() { - return 20 * time.Second +// ScaleTimeout scales an integration timeout for the running environment: the +// nested-docker nix-VM checks are slower than a host docker daemon and set +// HEADSCALE_INTEGRATION_TIMEOUT_SCALE to widen every budget; otherwise CI uses +// 2x and local dev the base. One knob covers exec, convergence, peer sync, and +// pool waits. Lives here (the lowest integration package) so every layer can +// reach it. +func ScaleTimeout(base time.Duration) time.Duration { + scale := os.Getenv("HEADSCALE_INTEGRATION_TIMEOUT_SCALE") + if scale != "" { + f, err := strconv.ParseFloat(scale, 64) + if err == nil && f > 0 { + return time.Duration(float64(base) * f) + } } - return 10 * time.Second + if util.IsCI() { + return base * 2 + } + + return base +} + +// defaultExecuteTimeout returns the timeout for docker exec commands. +func defaultExecuteTimeout() time.Duration { + return ScaleTimeout(10 * time.Second) } var ( diff --git a/integration/dsic/dsic.go b/integration/dsic/dsic.go index c165eee10..3d531fa7e 100644 --- a/integration/dsic/dsic.go +++ b/integration/dsic/dsic.go @@ -117,9 +117,14 @@ func (dsic *DERPServerInContainer) buildEntrypoint(derperArgs string) []string { // Wait for network to be ready commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done") - // Wait for TLS cert to be written (always written after container start) + // Wait for the TLS cert AND key to be written (both written after container + // start). Waiting only for the cert races: once it lands derper starts, and + // since it also needs the key — written a moment later — it exits and kills + // the container before the key upload (reliably, with slow nested-docker exec). commands = append(commands, fmt.Sprintf("while [ ! -f %s/%s.crt ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname)) + commands = append(commands, + fmt.Sprintf("while [ ! -f %s/%s.key ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname)) // If CA certs are configured, wait for them to be written if len(dsic.caCerts) > 0 { @@ -221,34 +226,56 @@ func New( var container *dockertest.Resource - buildOptions := &dockertest.BuildOptions{ - Dockerfile: "Dockerfile.derper", - ContextDir: dockerContextPath, - BuildArgs: []docker.BuildArg{}, - } - - switch version { - case "head": - buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ - Name: "VERSION_BRANCH", - Value: "main", - }) - default: - buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ - Name: "VERSION_BRANCH", - Value: "v" + version, - }) - } // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(runOptions, "derp") - container, err = pool.BuildAndRunWithBuildOptions( - buildOptions, - runOptions, - dockertestutil.DockerRestartPolicy, - dockertestutil.DockerAllowLocalIPv6, - dockertestutil.DockerAllowNetworkAdministration, - ) + // In CI / nix checks a prebuilt derper image is provided so we neither build + // the image nor clone tailscale at runtime, mirroring the tailscale and + // headscale image injection. Only the head image is prebuilt. + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_DERPER_IMAGE") + if err != nil { + return nil, err + } + + if prebuilt && version == "head" { + runOptions.Repository = repo + runOptions.Tag = tag + + container, err = pool.RunWithOptions( + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + } else { + buildOptions := &dockertest.BuildOptions{ + Dockerfile: "Dockerfile.derper", + ContextDir: dockerContextPath, + BuildArgs: []docker.BuildArg{}, + } + + switch version { + case "head": + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "main", + }) + default: + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "v" + version, + }) + } + + container, err = pool.BuildAndRunWithBuildOptions( + buildOptions, + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + } + if err != nil { return nil, fmt.Errorf( "%s starting tailscale DERPer container (version: %s): %w", diff --git a/integration/general_test.go b/integration/general_test.go index 9bdfd8c71..689c9a308 100644 --- a/integration/general_test.go +++ b/integration/general_test.go @@ -483,16 +483,7 @@ func TestTaildrop(t *testing.T) { _, err = scenario.ListTailscaleClientsFQDNs() requireNoErrListFQDN(t, err) - // Install curl on all clients - for _, client := range allClients { - if !strings.Contains(client.Hostname(), "head") { - command := []string{"apk", "add", "curl"} - _, _, err := client.Execute(command) - if err != nil { - t.Fatalf("failed to install curl on %s, err: %s", client.Hostname(), err) - } - } - } + // curl is baked into every image (no runtime install). // Helper to get FileTargets for a client. getFileTargets := func(client TailscaleClient) ([]apitype.FileTarget, error) { diff --git a/integration/helpers.go b/integration/helpers.go index 908c7447b..859d805fd 100644 --- a/integration/helpers.go +++ b/integration/helpers.go @@ -22,6 +22,7 @@ import ( policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/hscontrol/util" + "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/integrationutil" "github.com/juanfont/headscale/integration/tsic" "github.com/oauth2-proxy/mockoidc" @@ -922,13 +923,7 @@ func assertCommandOutputContains(t *testing.T, c TailscaleClient, command []stri // Uses longer timeouts in CI environments to account for slower resource allocation // and higher system load during automated testing. func dockertestMaxWait() time.Duration { - wait := 300 * time.Second //nolint - - if util.IsCI() { - wait = 600 * time.Second //nolint - } - - return wait + return dockertestutil.ScaleTimeout(300 * time.Second) } // didClientUseWebsocketForDERP analyzes client logs to determine if WebSocket was used for DERP. diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index d37bac9af..58de979c0 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -57,10 +57,8 @@ const ( ) var ( - errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") - errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag") - errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI") - errInvalidPostgresImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_POSTGRES_IMAGE format, expected repository:tag") + errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") + errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI") ) type fileInContainer struct { @@ -406,12 +404,12 @@ func New( pgRepo := "postgres" pgTag := "latest" - if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_POSTGRES_IMAGE"); prebuiltImage != "" { - repo, tag, found := strings.Cut(prebuiltImage, ":") - if !found { - return nil, errInvalidPostgresImageFormat - } + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_POSTGRES_IMAGE") + if err != nil { + return nil, err + } + if prebuilt { pgRepo = repo pgTag = tag } @@ -510,16 +508,13 @@ func New( var container *dockertest.Resource // Check if a pre-built image is available via environment variable - prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") - - if prebuiltImage != "" { - log.Printf("Using pre-built headscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value - // Parse image into repository and tag - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, errInvalidHeadscaleImageFormat - } + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") + if err != nil { + return nil, err + } + if prebuilt { + log.Printf("Using pre-built headscale image: %s:%s", repo, tag) //nolint:gosec // G706: integration-only log of trusted env value runOptions.Repository = repo runOptions.Tag = tag @@ -530,7 +525,7 @@ func New( dockertestutil.DockerAllowNetworkAdministration, ) if err != nil { - return nil, fmt.Errorf("running pre-built headscale container %q: %w", prebuiltImage, err) + return nil, fmt.Errorf("running pre-built headscale container %s:%s: %w", repo, tag, err) } } else if util.IsCI() { return nil, errHeadscaleImageRequiredInCI diff --git a/integration/integrationutil/util.go b/integration/integrationutil/util.go index a70145e0c..f31357c75 100644 --- a/integration/integrationutil/util.go +++ b/integration/integrationutil/util.go @@ -8,28 +8,61 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "fmt" "io" "math/big" "path/filepath" + "strings" "time" "github.com/juanfont/headscale/hscontrol/types" - "github.com/juanfont/headscale/hscontrol/util" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/tailcfg" ) -// PeerSyncTimeout returns the timeout for peer synchronization based on environment: -// 60s for dev, 120s for CI. -func PeerSyncTimeout() time.Duration { - if util.IsCI() { - return 120 * time.Second +var errInvalidImageFormat = errors.New("integration image env must be in repository:tag format") + +// PrebuiltImage reads a HEADSCALE_INTEGRATION_*_IMAGE knob (the full var name, +// e.g. "HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") and splits it into repository +// and tag. The bool is false when the knob is unset — the suite then builds the +// image itself; it errors when the value is set but is not "repository:tag". +// This is the one place the prebuilt-image knobs (used by the CI / nix-check +// path) are read, via tailscale's envknob. +func PrebuiltImage(envVar string) (string, string, bool, error) { + image := envknob.String(envVar) + if image == "" { + return "", "", false, nil } - return 60 * time.Second + repo, tag, err := ParseImageRef(image) + if err != nil { + return "", "", false, fmt.Errorf("%s=%w", envVar, err) + } + + return repo, tag, true, nil +} + +// ParseImageRef splits an already-resolved "repository:tag" image string into +// its parts. Use it where the image comes from somewhere other than a single +// knob (e.g. a websocket-tagged override chosen at runtime); for the common +// read-a-knob case use [PrebuiltImage]. +func ParseImageRef(image string) (string, string, error) { + repo, tag, found := strings.Cut(image, ":") + if !found { + return "", "", fmt.Errorf("%q: %w", image, errInvalidImageFormat) + } + + return repo, tag, nil +} + +// PeerSyncTimeout returns the timeout for peer synchronization: 60s for dev, +// scaled for CI / the slow nix VM. +func PeerSyncTimeout() time.Duration { + return dockertestutil.ScaleTimeout(60 * time.Second) } // PeerSyncRetryInterval returns the retry interval for peer synchronization checks. @@ -37,16 +70,10 @@ func PeerSyncRetryInterval() time.Duration { return 100 * time.Millisecond } -// ScaledTimeout returns the given timeout, scaled for CI environments -// where resource contention causes slower state propagation. -// Uses a 2x multiplier, consistent with PeerSyncTimeout (60s/120s) -// and dockertestMaxWait (300s/600s). +// ScaledTimeout returns the given convergence budget, scaled for the running +// environment via [dockertestutil.ScaleTimeout]. func ScaledTimeout(d time.Duration) time.Duration { - if util.IsCI() { - return d * 2 - } - - return d + return dockertestutil.ScaleTimeout(d) } func WriteFileToContainer( diff --git a/integration/scenario.go b/integration/scenario.go index 7af281619..58cbeb72d 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -3,6 +3,7 @@ package integration import ( "context" "crypto/tls" + _ "embed" "encoding/json" "errors" "fmt" @@ -22,7 +23,6 @@ import ( "time" clientv1 "github.com/juanfont/headscale/gen/client/v1" - "github.com/juanfont/headscale/hscontrol/capver" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/dsic" @@ -54,30 +54,38 @@ var ( errNoHeadscaleAvailable = errors.New("no headscale available") errNoUserAvailable = errors.New("no user available") errNoClientFound = errors.New("client not found") - - // AllVersions represents a list of Tailscale versions the suite - // uses to test compatibility with the [ControlServer]. - // - // The list contains two special cases, "head" and "unstable" which - // points to the current tip of Tailscale's main branch and the latest - // released unstable version. - // - // The rest of the version represents Tailscale versions that can be - // found in Tailscale's apt repository. - AllVersions = append([]string{"head", "unstable"}, capver.TailscaleLatestMajorMinor(capver.SupportedMajorMinorVersions, true)...) - - // MustTestVersions is the minimum set of versions we should test. - // At the moment, this is arbitrarily chosen as: - // - // - Two unstable (HEAD and unstable) - // - Two latest versions - // - Two oldest supported version. - MustTestVersions = append( - AllVersions[0:4], - AllVersions[len(AllVersions)-2:]..., - ) ) +// Regenerate tailscale-versions.json from capver + nix-prefetch-docker. Runs in +// the same `make generate` / `go generate ./...` pass as capver (and needs the +// nix dev shell); already-pinned tags are reused, so an unchanged set is a +// no-op. Refresh just the pins with `go generate ./integration/...`. +//go:generate go run ../tools/tailscale-versions + +//go:embed tailscale-versions.json +var tailscaleVersionsJSON []byte + +// MustTestVersions is the set of Tailscale versions the suite tests for +// compatibility with the [ControlServer]. It contains "head" (built from +// source) and "unstable", plus the latest and oldest supported releases. It is +// read from the generated tailscale-versions.json — the single source shared +// with the nix image pins (regenerate with `update-integration-images`, which +// derives the list from capver). +var MustTestVersions = mustTailscaleVersions() + +func mustTailscaleVersions() []string { + var v struct { + Versions []string `json:"versions"` + } + + err := json.Unmarshal(tailscaleVersionsJSON, &v) + if err != nil { + panic("parsing tailscale-versions.json: " + err.Error()) + } + + return v.Versions +} + // User represents a User in the [ControlServer] and a map of [TailscaleClient]'s // associated with the User. type User struct { @@ -1583,6 +1591,33 @@ const ( var errStatusCodeNotOK = errors.New("status code not OK") +// runHeadscaleImageContainer starts runOptions from the prebuilt headscale +// image when HEADSCALE_INTEGRATION_HEADSCALE_IMAGE is set, otherwise it builds +// Dockerfile.integration. The mock OIDC provider (headscale mockoidc) and the +// webservice (python3 -m http.server) both run on the headscale image, so they +// share the same prebuilt-or-build path hsic uses for headscale itself — which +// is what lets them come up offline (e.g. in the nix VM checks). +func (s *Scenario) runHeadscaleImageContainer(runOptions *dockertest.RunOptions) (*dockertest.Resource, error) { + repo, tag, prebuilt, err := integrationutil.PrebuiltImage("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") + if err != nil { + return nil, err + } + + if prebuilt { + runOptions.Repository = repo + runOptions.Tag = tag + + return s.pool.RunWithOptions(runOptions, dockertestutil.DockerRestartPolicy) + } + + buildOptions := &dockertest.BuildOptions{ + Dockerfile: hsic.IntegrationTestDockerFileName, + ContextDir: dockerContextPath, + } + + return s.pool.BuildAndRunWithBuildOptions(buildOptions, runOptions, dockertestutil.DockerRestartPolicy) +} + func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUser) error { port, err := dockertestutil.RandomFreeHostPort() if err != nil { @@ -1618,11 +1653,6 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse }, } - headscaleBuildOptions := &dockertest.BuildOptions{ - Dockerfile: hsic.IntegrationTestDockerFileName, - ContextDir: dockerContextPath, - } - err = s.pool.RemoveContainerByName(hostname) if err != nil { return err @@ -1633,11 +1663,7 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(mockOidcOptions, "oidc") - if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( //nolint:noinlineerr - headscaleBuildOptions, - mockOidcOptions, - dockertestutil.DockerRestartPolicy, - ); err == nil { + if pmockoidc, err := s.runHeadscaleImageContainer(mockOidcOptions); err == nil { //nolint:noinlineerr s.mockOIDC.r = pmockoidc } else { return err @@ -1722,16 +1748,7 @@ func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) { // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(webOpts, "web") - webBOpts := &dockertest.BuildOptions{ - Dockerfile: hsic.IntegrationTestDockerFileName, - ContextDir: dockerContextPath, - } - - web, err := s.pool.BuildAndRunWithBuildOptions( - webBOpts, - webOpts, - dockertestutil.DockerRestartPolicy, - ) + web, err := s.runHeadscaleImageContainer(webOpts) if err != nil { return nil, err } diff --git a/integration/ssh_test.go b/integration/ssh_test.go index 7ed50c206..fb14f8cd4 100644 --- a/integration/ssh_test.go +++ b/integration/ssh_test.go @@ -47,7 +47,6 @@ func sshScenario(t *testing.T, policy *policyv2.Policy, testName string, clients // to not configure DNS. tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(policy), @@ -445,7 +444,7 @@ func doSSHWithRetryAsUser( peerFQDN, _ := peer.FQDN() command := []string{ - "/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1", + "ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1", fmt.Sprintf("%s@%s", sshUser, peerFQDN), "'hostname'", } @@ -661,7 +660,7 @@ func doSSHCheckWithTimeout( peerFQDN, _ := peer.FQDN() command := []string{ - "/usr/bin/ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30", + "ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=30", fmt.Sprintf("%s@%s", "ssh-it-user", peerFQDN), "'hostname'", } @@ -935,7 +934,6 @@ func TestSSHOneUserToOneCheckModeOIDC(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(sshCheckPolicy()), @@ -1035,7 +1033,6 @@ func TestSSHCheckModeUnapprovedTimeout(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser ssh-it-user"), tsic.WithDockerWorkdir("/"), }, hsic.WithACLPolicy(sshCheckPolicy()), @@ -1607,7 +1604,6 @@ func TestSSHLocalpart(t *testing.T) { tsic.WithSSH(), tsic.WithNetfilter("off"), tsic.WithPackages("openssh"), - tsic.WithExtraCommands("adduser user1", "adduser user2"), tsic.WithDockerWorkdir("/"), }, hsic.WithTestName("sshlocalpart"), diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 3e19d2b14..6770c837e 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -26,6 +26,7 @@ import ( "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store/mem" @@ -65,7 +66,6 @@ var ( errTailscaleWrongPeerCount = errors.New("wrong peer count") errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey") errInvalidClientConfig = errors.New("verifiably invalid client config requested") - errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag") errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version") errContainerNotInitialized = errors.New("container not initialized") errFQDNNotYetAvailable = errors.New("FQDN not yet available") @@ -244,20 +244,33 @@ func WithAcceptRoutes() Option { } } -// WithPackages specifies Alpine packages to install when the container starts. -// This requires internet access and uses `apk add`. Common packages: +// WithPackages declares the tools a test needs inside the container. The images +// bake these in (no runtime install), so this asserts they are present at +// startup and fails the container loudly if not. Common packages: // - "python3" for HTTP server // - "curl" for HTTP client // - "bind-tools" for dig command -// - "iptables", "ip6tables" for firewall rules -// Note: Tests using this option require internet access and cannot use -// the built-in DERP server in offline mode. +// - "openssh" for the ssh client. func WithPackages(packages ...string) Option { return func(tsic *TailscaleInContainer) { tsic.withPackages = append(tsic.withPackages, packages...) } } +// packageBinary maps an alpine package name to a representative binary the +// images must provide, so [WithPackages] can assert the binary is on PATH +// rather than installing the package at runtime. +func packageBinary(pkg string) string { + switch pkg { + case "openssh": + return "ssh" + case "bind-tools": + return "dig" + default: + return pkg + } +} + // WithWebserver starts a Python HTTP server on the specified port // alongside tailscaled. This is useful for testing subnet routing // and ACL connectivity. Automatically adds "python3" to packages if needed. @@ -285,20 +298,31 @@ func (t *TailscaleInContainer) buildEntrypoint() []string { commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done") // If CA certs are configured, wait for them to be written by the Go code - // (certs are written after container start via [TailscaleInContainer.WriteFile]) - if len(t.caCerts) > 0 { + // (certs are written after container start via [TailscaleInContainer.WriteFile]). + // Wait for the LAST cert (highest index): they are written in order, so once + // user-{N-1}.crt exists every earlier one does too. Waiting only for user-0 + // races update-ca-certificates ahead of later certs (e.g. the headscale CA at + // user-1 when a DERP cert occupies user-0), permanently skipping their trust. + if n := len(t.caCerts); n > 0 { commands = append(commands, - fmt.Sprintf("while [ ! -f %s/user-0.crt ]; do sleep 0.1; done", caCertRoot)) + fmt.Sprintf("while [ ! -f %s/user-%d.crt ]; do sleep 0.1; done", caCertRoot, n-1)) } - // Install packages if requested (requires internet access) + // The images bake in every tool the suite needs (nix/images.nix for the head + // and version images, the alpine base otherwise), so assert the requested + // packages are present rather than installing them — a hermetic/offline run + // has no package mirror. Fail the container loudly if one is missing so an + // image regression is obvious in its log. packages := t.withPackages if t.withWebserverPort > 0 && !slices.Contains(packages, "python3") { packages = append(packages, "python3") } - if len(packages) > 0 { - commands = append(commands, "apk add --no-cache "+strings.Join(packages, " ")) + for _, pkg := range packages { + bin := packageBinary(pkg) + commands = append(commands, fmt.Sprintf( + "command -v %s >/dev/null 2>&1 || { echo 'tsic: required tool %s (package %s) missing from image' >&2; exit 1; }", + bin, bin, pkg)) } // Update CA certificates @@ -427,24 +451,31 @@ func New( switch version { case VersionHead: // Check if a pre-built image is available via environment variable - prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE") + prebuiltImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE") - // If custom build tags are required (e.g., for websocket DERP), we cannot use - // the pre-built image as it won't have the necessary code compiled in. + // If custom build tags are required (e.g., for websocket DERP), the + // default pre-built image won't have the necessary code compiled in. A + // websocket-tagged prebuilt image can be injected separately (CI / nix + // checks build one with ts_debug_websockets); otherwise fall back to + // building the image with the tags. hasBuildTags := len(tsic.buildConfig.tags) > 0 if hasBuildTags && prebuiltImage != "" { - log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value - prebuiltImage, tsic.buildConfig.tags) - prebuiltImage = "" + wsImage := envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_WEBSOCKET_IMAGE") + if tsic.withWebsocketDERP && wsImage != "" { + prebuiltImage = wsImage + } else { + log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", //nolint:gosec // G706: integration-only log of trusted env value + prebuiltImage, tsic.buildConfig.tags) + prebuiltImage = "" + } } if prebuiltImage != "" { log.Printf("Using pre-built tailscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value - // Parse image into repository and tag - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, errInvalidTailscaleImageFormat + repo, tag, err := integrationutil.ParseImageRef(prebuiltImage) + if err != nil { + return nil, err } tailscaleOptions.Repository = repo diff --git a/integration/tsric/tsric.go b/integration/tsric/tsric.go index e160d855d..fdeef4942 100644 --- a/integration/tsric/tsric.go +++ b/integration/tsric/tsric.go @@ -12,13 +12,13 @@ import ( "fmt" "io" "log" - "os" "strings" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "tailscale.com/envknob" "tailscale.com/util/rands" ) @@ -35,7 +35,7 @@ const ( // getPrebuiltImage returns the pre-built tailscale-rs Docker image name if set. func getPrebuiltImage() string { - return os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE") + return envknob.String("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE") } // TailscaleRustInContainer runs the tailscale-rs axum example as an @@ -210,9 +210,9 @@ func New( if prebuiltImage := getPrebuiltImage(); prebuiltImage != "" { log.Printf("Using pre-built tailscale-rs image: %s", prebuiltImage) - repo, tag, ok := strings.Cut(prebuiltImage, ":") - if !ok { - return nil, fmt.Errorf("tsric: invalid image format %q, expected repository:tag", prebuiltImage) //nolint:err113 + repo, tag, err := integrationutil.ParseImageRef(prebuiltImage) + if err != nil { + return nil, fmt.Errorf("tsric: %w", err) } runOptions.Repository = repo