feat(cli): build deployment images on prebuilt base images

.changeset/prebuilt-base-images.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Deployment builds now use custom base layer images and no longer install system packages during every build. This improves layer caching resulting in both faster deployments and faster image pulls on the worker cluster side.

packages/cli-v3/src/deploy/buildImage.test.ts

@@ -1,20 +1,36 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
 import type { BuildRuntime } from "@trigger.dev/core/v3/schemas";
 import { describe, expect, it } from "vitest";
-import { generateContainerfile } from "./buildImage.js";
+import {
+  BASE_IMAGE,
+  BUILD_IMAGE,
+  DEFAULT_PACKAGES,
+  TOOLCHAIN_PACKAGES,
+  generateContainerfile,
+} from "./buildImage.js";
 
-const nodeImages: Array<[BuildRuntime, string]> = [
+const images: Array<[BuildRuntime, string, string]> = [
   [
     "node-24",
-    "node:24.18.0-bookworm-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d",
+    "triggerdotdev/node:24-bookworm@sha256:d2d0c01822409f6d2de1cc69a9e718424048fa12fc64b223dfa84f6db44d0ffd",
+    "triggerdotdev/node:24-bookworm-build@sha256:19322289508ae9b4be0b769acac179637e4b57d363844682f4b116feb951267d",
   ],
   [
     "node-26",
-    "node:26.4.0-bookworm-slim@sha256:ec82d089a8ae2cf02628da7b34ea57dc357b24db724d557fe2d240e6beb659c1",
+    "triggerdotdev/node:26-bookworm@sha256:0e9b19f814f32d3766a8cf167835a499349df5bd34702611577b1f75bc2d2026",
+    "triggerdotdev/node:26-bookworm-build@sha256:de5cdfd683dabad582182c79779135d59faac0e6893b6cf04520d5a0dc826dd7",
+  ],
+  [
+    "bun",
+    "triggerdotdev/bun:1.3-node20-bookworm@sha256:25b467196277b9d75a37773ee36d28b65ca81a6f41786f7d7d7f1fad95fb5a31",
+    "triggerdotdev/bun:1.3-node20-bookworm-build@sha256:a2d5e6d1ec25946ca1d86abdd9ffb9c589376df64ee1b490c461607953931245",
   ],
 ];
 
 describe("generateContainerfile", () => {
-  it.each(nodeImages)("selects the pinned multiplatform image for %s", async (runtime, image) => {
+  it.each(images)("uses the pinned published base images for %s", async (runtime, base, build) => {
     const containerfile = await generateContainerfile({
       runtime,
       build: {},
@@ -23,7 +39,108 @@ describe("generateContainerfile", () => {
       entrypoint: "entrypoint.js",
     });
 
-    expect(containerfile).toContain(`FROM ${image} AS base`);
+    expect(containerfile).toContain(`FROM ${base} AS base`);
+    expect(containerfile).toContain(`FROM ${build} AS build`);
+  });
+
+  it.each(["node", "bun"] as BuildRuntime[])(
+    "runs no package installation for uncustomized projects on %s",
+    async (runtime) => {
+      const containerfile = await generateContainerfile({
+        runtime,
+        build: {},
+        image: undefined,
+        indexScript: "index.js",
+        entrypoint: "entrypoint.js",
+      });
+
+      expect(containerfile).not.toContain("apt-get");
+      expect(containerfile).not.toContain("FROM base AS build");
+    }
+  );
+
+  it("pairs every runtime image with its -build variant", () => {
+    for (const runtime of Object.keys(BASE_IMAGE) as BuildRuntime[]) {
+      const baseRef = BASE_IMAGE[runtime].split("@")[0];
+      const buildRef = BUILD_IMAGE[runtime].split("@")[0];
+
+      expect(buildRef).toBe(`${baseRef}-build`);
+      expect(BASE_IMAGE[runtime]).toMatch(/@sha256:[a-f0-9]{64}$/);
+      expect(BUILD_IMAGE[runtime]).toMatch(/@sha256:[a-f0-9]{64}$/);
+    }
+  });
+
+  it("matches the published base image package lists", () => {
+    const imagesJson = JSON.parse(
+      readFileSync(
+        join(fileURLToPath(import.meta.url), "../../../../../base-images/images.json"),
+        "utf-8"
+      )
+    );
+
+    expect(imagesJson.packages.split(" ").sort()).toEqual([...DEFAULT_PACKAGES].sort());
+    expect(imagesJson.buildPackages).toBe(TOOLCHAIN_PACKAGES);
+  });
+
+  it("applies instructions once and builds FROM base when they are present", async () => {
+    const containerfile = await generateContainerfile({
+      runtime: "node-22",
+      build: {},
+      image: {
+        pkgs: ["jq", "curl", "git"],
+        instructions: ["RUN echo custom > /etc/marker"],
+      },
+      indexScript: "index.js",
+      entrypoint: "entrypoint.js",
+    });
+
+    // sorted, defaults filtered out, downgrades allowed for pinned defaults,
+    // and repaired first: apt-get install refuses to run on dpkg state broken
+    // by an instruction
+    const installRun = `apt-get --fix-broken install -y --no-install-recommends && \\
+  apt-get install -y --no-install-recommends --allow-downgrades curl jq`;
+    const first = containerfile.indexOf(installRun);
+
+    expect(first).toBeGreaterThan(containerfile.indexOf("RUN echo custom > /etc/marker"));
+    expect(containerfile.indexOf(installRun, first + 1)).toBe(-1);
+    expect(containerfile).toContain("FROM base AS build");
+    expect(containerfile).toContain(TOOLCHAIN_PACKAGES);
+  });
+
+  it("keeps the prebuilt toolchain image for package-only projects", async () => {
+    const containerfile = await generateContainerfile({
+      runtime: "node-22",
+      build: {},
+      image: { pkgs: ["jq"] },
+      indexScript: "index.js",
+      entrypoint: "entrypoint.js",
+    });
+
+    const installLine = "apt-get install -y --no-install-recommends --allow-downgrades jq";
+    const first = containerfile.indexOf(installLine);
+
+    // installed in both stages, since base and build are separate images
+    expect(first).toBeGreaterThan(-1);
+    expect(containerfile.indexOf(installLine, first + 1)).toBeGreaterThan(first);
+    expect(containerfile).not.toContain("FROM base AS build");
+    // a pristine base has nothing to repair
+    expect(containerfile).not.toContain("--fix-broken");
+  });
+
+  it("repairs dpkg state after instructions when there are no user packages", async () => {
+    const containerfile = await generateContainerfile({
+      runtime: "node-22",
+      build: {},
+      image: { instructions: ["RUN echo custom > /etc/marker"] },
+      indexScript: "index.js",
+      entrypoint: "entrypoint.js",
+    });
+
+    const instructions = containerfile.indexOf("RUN echo custom > /etc/marker");
+    const repair = containerfile.indexOf("apt-get --fix-broken install -y");
+
+    expect(instructions).toBeGreaterThan(-1);
+    expect(repair).toBeGreaterThan(instructions);
   });
 
   it.each(["node", "bun"] as BuildRuntime[])(

packages/cli-v3/src/deploy/buildImage.ts

@@ -687,18 +687,31 @@ export type GenerateContainerfileOptions = {
   entrypoint: string;
 };
 
-const BASE_IMAGE: Record<BuildRuntime, string> = {
-  bun: "imbios/bun-node:1.3.3-20-slim@sha256:59d84856a7e31eec83afedadb542f7306f672343b8b265c70d733404a6e8834b",
-  node: "node:21.7.3-bookworm-slim@sha256:dfc05dee209a1d7adf2ef189bd97396daad4e97c6eaa85778d6f75205ba1b0fb",
+// Prebuilt in base-images/; both maps must be bumped together, from one publish run
+export const BASE_IMAGE: Record<BuildRuntime, string> = {
+  bun: "triggerdotdev/bun:1.3-node20-bookworm@sha256:25b467196277b9d75a37773ee36d28b65ca81a6f41786f7d7d7f1fad95fb5a31",
+  node: "triggerdotdev/node:21-bookworm@sha256:49c6575cda32f63ac21a4aeaabc360dc50c6f767b86b675b44de7a9e9b6ca3fc",
   "node-22":
-    "node:22.16.0-bookworm-slim@sha256:048ed02c5fd52e86fda6fbd2f6a76cf0d4492fd6c6fee9e2c463ed5108da0e34",
+    "triggerdotdev/node:22-bookworm@sha256:3d1b59a1d50c3df713078a7b18386441cf7fdbaeea6da799247df5a2e180bdd5",
   "node-24":
-    "node:24.18.0-bookworm-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d",
+    "triggerdotdev/node:24-bookworm@sha256:d2d0c01822409f6d2de1cc69a9e718424048fa12fc64b223dfa84f6db44d0ffd",
   "node-26":
-    "node:26.4.0-bookworm-slim@sha256:ec82d089a8ae2cf02628da7b34ea57dc357b24db724d557fe2d240e6beb659c1",
+    "triggerdotdev/node:26-bookworm@sha256:0e9b19f814f32d3766a8cf167835a499349df5bd34702611577b1f75bc2d2026",
 };
 
-const DEFAULT_PACKAGES = ["busybox", "ca-certificates", "dumb-init", "git", "openssl"];
+export const BUILD_IMAGE: Record<BuildRuntime, string> = {
+  bun: "triggerdotdev/bun:1.3-node20-bookworm-build@sha256:a2d5e6d1ec25946ca1d86abdd9ffb9c589376df64ee1b490c461607953931245",
+  node: "triggerdotdev/node:21-bookworm-build@sha256:98f2bc6beb124da3c3aa9abba587a6c3d08a1aada217b5bb91f843364184d1d0",
+  "node-22":
+    "triggerdotdev/node:22-bookworm-build@sha256:acc6f0143021f532b601bf9fa2cd7745b07612358f94acb8e1cd864468320a81",
+  "node-24":
+    "triggerdotdev/node:24-bookworm-build@sha256:19322289508ae9b4be0b769acac179637e4b57d363844682f4b116feb951267d",
+  "node-26":
+    "triggerdotdev/node:26-bookworm-build@sha256:de5cdfd683dabad582182c79779135d59faac0e6893b6cf04520d5a0dc826dd7",
+};
+
+// Preinstalled in the published base images; must match base-images/images.json
+export const DEFAULT_PACKAGES = ["busybox", "ca-certificates", "dumb-init", "git", "openssl"];
 
 export async function generateContainerfile(options: GenerateContainerfileOptions) {
   switch (options.runtime) {
@@ -714,6 +727,31 @@ export async function generateContainerfile(options: GenerateContainerfileOption
   }
 }
 
+// repair: apt-get install refuses to run on dpkg state an instruction left
+// broken (the dpkg -i pattern). --allow-downgrades: a user pin of a
+// preinstalled package is a downgrade by the time this runs.
+function aptInstall(packages: string[], { repair }: { repair: boolean }): string {
+  const repairStep = repair
+    ? `apt-get --fix-broken install -y --no-install-recommends && \\
+  `
+    : "";
+
+  return `RUN apt-get update && \\
+  ${repairStep}apt-get install -y --no-install-recommends --allow-downgrades ${packages.join(" ")} && \\
+  apt-get clean && \\
+  rm -rf /var/lib/apt/lists/*`;
+}
+
+function aptRepair(): string {
+  return `RUN apt-get update && \\
+  apt-get --fix-broken install -y --no-install-recommends && \\
+  apt-get clean && \\
+  rm -rf /var/lib/apt/lists/*`;
+}
+
+// Must match base-images/images.json buildPackages, which the -build images preinstall
+export const TOOLCHAIN_PACKAGES = "python3 make g++";
+
 const parseGenerateOptions = (options: GenerateContainerfileOptions) => {
   const buildArgs = Object.entries(options.build.env || {})
     .flatMap(([key]) => `ARG ${key}`)
@@ -726,43 +764,59 @@ const parseGenerateOptions = (options: GenerateContainerfileOptions) => {
   const postInstallCommands = (options.build.commands || []).map((cmd) => `RUN ${cmd}`).join("\n");
 
   const baseInstructions = (options.image?.instructions || []).join("\n");
-  const packages = Array.from(new Set(DEFAULT_PACKAGES.concat(options.image?.pkgs || []))).join(
-    " "
-  );
+  const userPackages = Array.from(new Set(options.image?.pkgs || []))
+    .filter((pkg) => !DEFAULT_PACKAGES.includes(pkg))
+    .sort();
+
+  const customization = [
+    baseInstructions,
+    userPackages.length > 0
+      ? aptInstall(userPackages, { repair: baseInstructions.length > 0 })
+      : baseInstructions
+        ? aptRepair()
+        : "",
+  ]
+    .filter(Boolean)
+    .join("\n\n");
+
+  // Instructions run once (FROM base) since their downloads are unbounded;
+  // package-only projects keep the prebuilt toolchain and repeat the small install
+  const buildStage = baseInstructions
+    ? `FROM base AS build
+
+RUN apt-get update && \\
+  apt-get install -y --no-install-recommends ${TOOLCHAIN_PACKAGES} && \\
+  apt-get clean && \\
+  rm -rf /var/lib/apt/lists/*`
+    : `FROM ${BUILD_IMAGE[options.runtime]} AS build
+
+ENV DEBIAN_FRONTEND=noninteractive${
+        userPackages.length > 0 ? `\n\n${aptInstall(userPackages, { repair: false })}` : ""
+      }`;
 
   return {
     baseImage: BASE_IMAGE[options.runtime],
-    baseInstructions,
+    buildStage,
+    customization,
     buildArgs,
     buildEnvVars,
-    packages,
     postInstallCommands,
   };
 };
 
 async function generateBunContainerfile(options: GenerateContainerfileOptions) {
-  const { baseImage, buildArgs, buildEnvVars, postInstallCommands, baseInstructions, packages } =
+  const { baseImage, buildStage, buildArgs, buildEnvVars, postInstallCommands, customization } =
     parseGenerateOptions(options);
 
   return `# syntax=docker/dockerfile:1
 # check=skip=SecretsUsedInArgOrEnv
 FROM ${baseImage} AS base
 
-${baseInstructions}
-
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-  apt-get --fix-broken install -y && \
-  apt-get install -y --no-install-recommends ${packages} && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
 
-FROM base AS build
+${customization}
 
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends python3 make g++ && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
+${buildStage}
 
 USER bun
 WORKDIR /app
@@ -853,28 +907,18 @@ CMD []
 }
 
 async function generateNodeContainerfile(options: GenerateContainerfileOptions) {
-  const { baseImage, buildArgs, buildEnvVars, postInstallCommands, baseInstructions, packages } =
+  const { baseImage, buildStage, buildArgs, buildEnvVars, postInstallCommands, customization } =
     parseGenerateOptions(options);
 
   return `# syntax=docker/dockerfile:1
 # check=skip=SecretsUsedInArgOrEnv
 FROM ${baseImage} AS base
 
-${baseInstructions}
-
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-  apt-get --fix-broken install -y && \
-  apt-get install -y --no-install-recommends ${packages} && \
-  apt-get clean && rm -rf /var/lib/apt/lists/*
-
-FROM base AS build
-
-# Install build dependencies
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends python3 make g++ && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
+
+${customization}
+
+${buildStage}
 
 USER node
 WORKDIR /app