diff --git a/.changeset/deterministic-apt-layer.md b/.changeset/deterministic-apt-layer.md new file mode 100644 index 0000000000..50d4b451f5 --- /dev/null +++ b/.changeset/deterministic-apt-layer.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Deployed images now install base system packages from a pinned Debian snapshot archive, so pushed images typically share one identical package layer across projects and builds instead of a near-duplicate per project, speeding up image pulls. Base package versions are frozen at the pinned snapshot date and move forward with CLI releases. Set TRIGGER_BUILD_SKIP_APT_SNAPSHOT=1 to fall back to the live package archive. diff --git a/docs/config/extensions/aptGet.mdx b/docs/config/extensions/aptGet.mdx index 817c4157d7..9549f42db9 100644 --- a/docs/config/extensions/aptGet.mdx +++ b/docs/config/extensions/aptGet.mdx @@ -32,3 +32,12 @@ export default defineConfig({ }, }); ``` + + + Packages install from a [Debian snapshot archive](https://snapshot.debian.org) pinned per CLI + version, so builds are reproducible and version pins keep working even after the live archive + moves on. This also means a pinned version must exist in that snapshot: a version published after + the CLI release's snapshot date won't resolve until you update the CLI. Set the + `TRIGGER_BUILD_SKIP_APT_SNAPSHOT=1` environment variable when running the deploy command to + install from the live Debian archive instead. + diff --git a/packages/cli-v3/src/deploy/buildImage.test.ts b/packages/cli-v3/src/deploy/buildImage.test.ts index cbeb58c083..ca2fa5b182 100644 --- a/packages/cli-v3/src/deploy/buildImage.test.ts +++ b/packages/cli-v3/src/deploy/buildImage.test.ts @@ -1,5 +1,5 @@ import type { BuildRuntime } from "@trigger.dev/core/v3/schemas"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { generateContainerfile } from "./buildImage.js"; const nodeImages: Array<[BuildRuntime, string]> = [ @@ -74,4 +74,116 @@ describe("generateContainerfile", () => { expect(rmNodeModules).toBeGreaterThan(codeStage); } ); + + describe("apt snapshot pinning", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it.each(["node", "bun"] as BuildRuntime[])( + "pins apt to the Debian snapshot archive and scrubs timestamped files for %s", + async (runtime) => { + const containerfile = await generateContainerfile({ + runtime, + build: {}, + image: undefined, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + expect(containerfile).toMatch( + /deb \[check-valid-until=no signed-by=\S+\] http:\/\/snapshot\.debian\.org\/archive\/debian\/\d{8}T\d{6}Z bookworm main/ + ); + // codename guard so a future non-bookworm base pin fails with an actionable error + expect(containerfile).toContain('[ "$VERSION_CODENAME" = "bookworm" ]'); + expect(containerfile).toContain("/archive/debian-security/"); + expect(containerfile).toContain("rm -f /etc/apt/sources.list.d/debian.sources"); + expect(containerfile).toContain("/var/log/dpkg.log"); + } + ); + + it("keeps the default package layer identical for customized projects", async () => { + const containerfile = await generateContainerfile({ + runtime: "node-22", + build: {}, + image: { + pkgs: ["jq", "curl"], + instructions: ["RUN echo custom > /etc/marker"], + }, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + const defaultInstall = containerfile.indexOf( + "apt-get install -y --no-install-recommends busybox ca-certificates dumb-init git openssl" + ); + const instructions = containerfile.indexOf("RUN echo custom > /etc/marker"); + // sorted, deduplicated, and separate from the default install line + const userInstall = containerfile.indexOf( + "apt-get install -y --no-install-recommends --allow-downgrades curl jq" + ); + + expect(defaultInstall).toBeGreaterThan(-1); + expect(instructions).toBeGreaterThan(defaultInstall); + expect(userInstall).toBeGreaterThan(instructions); + // the user install reuses the sources written by the default install + expect(containerfile.slice(userInstall)).not.toContain("snapshot.debian.org"); + }); + + it("drops the snapshot pin when TRIGGER_BUILD_SKIP_APT_SNAPSHOT is set", async () => { + vi.stubEnv("TRIGGER_BUILD_SKIP_APT_SNAPSHOT", "1"); + + const containerfile = await generateContainerfile({ + runtime: "node-22", + build: {}, + image: undefined, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + expect(containerfile).not.toContain("snapshot.debian.org"); + // the base-stage default install must still be present, from the live archive + expect(containerfile).toContain( + "apt-get install -y --no-install-recommends busybox ca-certificates dumb-init git openssl" + ); + }); + + it("repairs dpkg state after instructions when there are no user packages", async () => { + const containerfile = await generateContainerfile({ + runtime: "node-22", + build: {}, + image: { instructions: ["RUN echo custom > /etc/marker"] }, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + const instructions = containerfile.indexOf("RUN echo custom > /etc/marker"); + const repair = containerfile.indexOf("apt-get --fix-broken install -y"); + + expect(instructions).toBeGreaterThan(-1); + expect(repair).toBeGreaterThan(instructions); + }); + + it("pins a snapshot no older than 90 days", async () => { + const containerfile = await generateContainerfile({ + runtime: "node-22", + build: {}, + image: undefined, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + const match = containerfile.match(/archive\/debian\/(\d{4})(\d{2})(\d{2})T/); + expect(match).not.toBeNull(); + + const [, year, month, day] = match!; + const snapshotAgeDays = + (Date.now() - Date.UTC(Number(year), Number(month) - 1, Number(day))) / 86_400_000; + + expect( + snapshotAgeDays, + "DEBIAN_SNAPSHOT is stale; deployed images are missing recent Debian security updates. Bump it in buildImage.ts." + ).toBeLessThan(90); + }); + }); }); diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts index 210a70be34..c5bb765000 100644 --- a/packages/cli-v3/src/deploy/buildImage.ts +++ b/packages/cli-v3/src/deploy/buildImage.ts @@ -700,6 +700,72 @@ const BASE_IMAGE: Record = { const DEFAULT_PACKAGES = ["busybox", "ca-certificates", "dumb-init", "git", "openssl"]; +// Freezes the Debian archive so the package layer is a pure function of this +// timestamp and the base image, making it byte-identical across projects. +// INVARIANT: must be at or after the archive state every BASE_IMAGE was built +// from, or apt hits unsatisfiable exact-version downgrades; bump it whenever +// a base pin is bumped (this is also how packages pick up security updates). +const DEBIAN_SNAPSHOT = "20260810T000000Z"; + +// Must match the Debian release of every BASE_IMAGE; the generated Containerfile +// asserts this at build time +const DEBIAN_SUITE = "bookworm"; + +// Files apt/dpkg write with wall-clock timestamps, which would break layer determinism +const APT_SCRUB = + "rm -rf /var/lib/apt/lists/* /var/log/dpkg.log /var/log/apt /var/log/alternatives.log /var/cache/ldconfig/aux-cache /var/cache/debconf/*-old"; + +function aptSourcesSetup(): string { + if (process.env.TRIGGER_BUILD_SKIP_APT_SNAPSHOT === "1") { + return ""; + } + + // check-valid-until=no: pinned Release files outlive their Valid-Until window. + // http, not https: the slim base images have no ca-certificates yet, and apt + // integrity comes from GPG-signed Release files rather than TLS. + const sourceOptions = + "[check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg]"; + const sources = [ + `deb ${sourceOptions} http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE} main`, + `deb ${sourceOptions} http://snapshot.debian.org/archive/debian-security/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE}-security main`, + `deb ${sourceOptions} http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE}-updates main`, + ]; + + return `. /etc/os-release && [ "$VERSION_CODENAME" = "${DEBIAN_SUITE}" ] || { echo "Base image is Debian $VERSION_CODENAME but the CLI pins ${DEBIAN_SUITE} apt sources. Set TRIGGER_BUILD_SKIP_APT_SNAPSHOT=1 to use the live archive."; exit 1; } && \\ + printf '%s\\n' ${sources.map((line) => `'${line}'`).join(" ")} > /etc/apt/sources.list && \\ + rm -f /etc/apt/sources.list.d/debian.sources && \\ + echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries && \\ + `; +} + +function aptInstall(packages: string[], { setupSources }: { setupSources: boolean }): string { + // fix-broken repairs dpkg state left by instructions (e.g. dpkg -i of a local + // .deb); pointless in the default install, which runs on a pristine base. + // --allow-downgrades: a user pin of a default package (e.g. openssl=) is + // a downgrade by the time the user install runs on top of the default layer. + const repair = setupSources + ? "" + : `apt-get --fix-broken install -y && \\ + `; + const installFlags = setupSources + ? "-y --no-install-recommends" + : "-y --no-install-recommends --allow-downgrades"; + + return `RUN ${setupSources ? aptSourcesSetup() : ""}apt-get update && \\ + ${repair}apt-get install ${installFlags} ${packages.join(" ")} && \\ + apt-get clean && \\ + ${APT_SCRUB}`; +} + +// Instructions can leave dpkg in a broken state that the user-packages install +// would normally repair; when there are no user packages, repair explicitly +function aptRepair(): string { + return `RUN apt-get update && \\ + apt-get --fix-broken install -y && \\ + apt-get clean && \\ + ${APT_SCRUB}`; +} + export async function generateContainerfile(options: GenerateContainerfileOptions) { switch (options.runtime) { case "node": @@ -726,36 +792,54 @@ const parseGenerateOptions = (options: GenerateContainerfileOptions) => { const postInstallCommands = (options.build.commands || []).map((cmd) => `RUN ${cmd}`).join("\n"); const baseInstructions = (options.image?.instructions || []).join("\n"); - const packages = Array.from(new Set(DEFAULT_PACKAGES.concat(options.image?.pkgs || []))).join( - " " - ); + + // Default packages install alone so their layer stays identical across + // projects; user packages and instructions only add layers on top + const defaultPackages = [...DEFAULT_PACKAGES].sort(); + const userPackages = Array.from(new Set(options.image?.pkgs || [])) + .filter((pkg) => !DEFAULT_PACKAGES.includes(pkg)) + .sort(); + + const defaultPackagesInstall = aptInstall(defaultPackages, { setupSources: true }); + const userPackagesInstall = + userPackages.length > 0 + ? aptInstall(userPackages, { setupSources: false }) + : baseInstructions.length > 0 + ? aptRepair() + : ""; return { baseImage: BASE_IMAGE[options.runtime], baseInstructions, buildArgs, buildEnvVars, - packages, + defaultPackagesInstall, + userPackagesInstall, postInstallCommands, }; }; async function generateBunContainerfile(options: GenerateContainerfileOptions) { - const { baseImage, buildArgs, buildEnvVars, postInstallCommands, baseInstructions, packages } = - parseGenerateOptions(options); + const { + baseImage, + buildArgs, + buildEnvVars, + postInstallCommands, + baseInstructions, + defaultPackagesInstall, + userPackagesInstall, + } = parseGenerateOptions(options); return `# syntax=docker/dockerfile:1 # check=skip=SecretsUsedInArgOrEnv FROM ${baseImage} AS base +ENV DEBIAN_FRONTEND=noninteractive +${defaultPackagesInstall} + ${baseInstructions} -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get --fix-broken install -y && \ - apt-get install -y --no-install-recommends ${packages} && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +${userPackagesInstall} FROM base AS build @@ -853,20 +937,26 @@ CMD [] } async function generateNodeContainerfile(options: GenerateContainerfileOptions) { - const { baseImage, buildArgs, buildEnvVars, postInstallCommands, baseInstructions, packages } = - parseGenerateOptions(options); + const { + baseImage, + buildArgs, + buildEnvVars, + postInstallCommands, + baseInstructions, + defaultPackagesInstall, + userPackagesInstall, + } = parseGenerateOptions(options); return `# syntax=docker/dockerfile:1 # check=skip=SecretsUsedInArgOrEnv FROM ${baseImage} AS base +ENV DEBIAN_FRONTEND=noninteractive +${defaultPackagesInstall} + ${baseInstructions} -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get --fix-broken install -y && \ - apt-get install -y --no-install-recommends ${packages} && \ - apt-get clean && rm -rf /var/lib/apt/lists/* +${userPackagesInstall} FROM base AS build