Improve build times for entire corpus

.github/workflows/site-preview.yml

@@ -1,6 +1,6 @@
-name: Deploy Nimbus site preview
+name: Deploy docs preview
 
-run-name: Preview Nimbus PR #${{ github.event.pull_request.number || inputs.pull_request_number }}
+run-name: Deploy docs preview for PR #${{ github.event.pull_request.number || inputs.pull_request_number }}
 
 on:
   pull_request_target:
@@ -30,6 +30,35 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
+      # A superseding run may start after GitHub terminated the previous runner
+      # before its EXIT trap completed. Clear that stale state before any
+      # validation in this run can fail.
+      - name: Clear an interrupted preview status
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || inputs.pull_request_number }}
+        run: |
+          marker="<!-- nimbus-site-preview -->"
+          comment_id="$({
+            gh api --paginate "repos/$GH_REPO/issues/$PULL_REQUEST_NUMBER/comments" \
+              --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | contains(\"$marker\")) | .id"
+          } | tail -n 1)"
+          if [[ -z "$comment_id" ]]; then
+            exit 0
+          fi
+
+          body="$(gh api "repos/$GH_REPO/issues/comments/$comment_id" --jq .body)"
+          if [[ "$body" != *"🟡 Building"* ]]; then
+            exit 0
+          fi
+
+          updated_at="$(date -u '+%Y-%m-%d %H:%M UTC')"
+          body="$(jq -nr --arg body "$body" --arg updated_at "$updated_at" \
+            '$body | gsub("🟡 Building"; "🔴 Failed") | sub("\\| [^|\\n]+ \\|$"; "| \($updated_at) |")')"
+          gh api --method PATCH "repos/$GH_REPO/issues/comments/$comment_id" -f body="$body" >/dev/null
+
       - name: Require a trusted manual invocation
         if: github.event_name == 'workflow_dispatch'
         env:
@@ -150,6 +179,9 @@ jobs:
           HEAD_SHA: ${{ steps.pull-request.outputs.head_sha }}
           PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || inputs.pull_request_number }}
           TRANSLATION_SCOPE: ${{ steps.translations.outputs.scope }}
+          TRANSLATION_SUMMARY: ${{ steps.translations.outputs.summary }}
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
           TRUST: ${{ steps.pull-request.outputs.trust }}
           VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
           VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
@@ -158,6 +190,60 @@ jobs:
           repository_owner="${GITHUB_REPOSITORY%%/*}"
           repository_name="${GITHUB_REPOSITORY#*/}"
 
+          update_preview_comment() {
+            local deployment_status="$1"
+            local updated_at="$(date -u '+%Y-%m-%d %H:%M UTC')"
+            local marker="<!-- nimbus-site-preview -->"
+            local docs_url="${preview_url%/}/docs"
+            local body
+            local comment_id
+            body="$(printf '🕵 %s\n\n| Preview | Deployment | Components | Updated (UTC) |\n| --- | --- | --- | --- |\n| [Docs preview](%s) | [%s](%s) | %s | %s |' \
+              "$marker" \
+              "$docs_url" \
+              "$deployment_status" \
+              "$preview_url" \
+              "$TRANSLATION_SUMMARY" \
+              "$updated_at")"
+            if ! comment_id="$({
+              gh api --paginate "repos/$GH_REPO/issues/$PULL_REQUEST_NUMBER/comments" \
+                --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | contains(\"$marker\")) | .id"
+            } | tail -n 1)"; then
+              return 1
+            fi
+            if [[ -n "$comment_id" ]]; then
+              gh api --method PATCH "repos/$GH_REPO/issues/comments/$comment_id" -f body="$body" >/dev/null
+            else
+              gh pr comment "$PULL_REQUEST_NUMBER" --body "$body"
+            fi
+          }
+
+          update_preview_comment_with_retry() {
+            local deployment_status="$1"
+            local attempt
+            for attempt in 1 2 3; do
+              if update_preview_comment "$deployment_status"; then
+                return 0
+              fi
+              echo "::warning::Unable to update the preview comment (attempt $attempt of 3)."
+              if (( attempt < 3 )); then
+                sleep 2
+              fi
+            done
+            return 1
+          }
+
+          preview_finished=false
+          finalize_preview_comment() {
+            local exit_code=$?
+            trap - EXIT
+            if [[ "$preview_finished" != true ]]; then
+              if ! update_preview_comment_with_retry "🔴 Failed"; then
+                echo "::warning::Unable to mark the preview comment as failed."
+              fi
+            fi
+            exit "$exit_code"
+          }
+
           if ! project_response="$(
             curl --silent --show-error --fail-with-body \
               --header "Authorization: Bearer $VERCEL_TOKEN" \
@@ -225,8 +311,20 @@ jobs:
           preview_url="https://${deployment_url#https://}"
           echo "preview_url=$preview_url" >> "$GITHUB_OUTPUT"
           echo "Vercel deployment: $preview_url"
+          if [[ ! "$preview_url" =~ ^https://[^[:space:]]+$ ]]; then
+            echo "Vercel returned an invalid preview URL." >&2
+            exit 1
+          fi
+          trap finalize_preview_comment EXIT
+          trap 'exit 130' INT
+          trap 'exit 143' TERM
+          if ! update_preview_comment_with_retry "🟡 Building"; then
+            echo "::warning::Unable to mark the preview comment as building."
+          fi
 
-          for _ in {1..360}; do
+          # Leave five minutes for the EXIT trap to update the comment before
+          # the 60-minute job timeout terminates the runner.
+          for _ in {1..330}; do
             if ! deployment_response="$(
               curl --silent --show-error --fail-with-body \
                 --header "Authorization: Bearer $VERCEL_TOKEN" \
@@ -260,30 +358,7 @@ jobs:
             echo "Timed out waiting for Vercel deployment $deployment_id." >&2
             exit 1
           fi
-          if [[ ! "$preview_url" =~ ^https://[^[:space:]]+$ ]]; then
-            echo "Vercel returned an invalid preview URL." >&2
-            exit 1
-          fi
-
-      - name: Add the preview link to the pull request
-        env:
-          GH_TOKEN: ${{ github.token }}
-          GH_REPO: ${{ github.repository }}
-          PREVIEW_URL: ${{ steps.deploy.outputs.preview_url }}
-          PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || inputs.pull_request_number }}
-          TRANSLATION_SUMMARY: ${{ steps.translations.outputs.summary }}
-          TRUST: ${{ steps.pull-request.outputs.trust }}
-        run: |
-          marker="<!-- nimbus-site-preview -->"
-          body="$(printf '%s\n%s' \
-            "$marker" \
-            "Nimbus documentation preview ($TRUST, $TRANSLATION_SUMMARY): $PREVIEW_URL")"
-          comment_id="$({
-            gh api --paginate "repos/$GH_REPO/issues/$PULL_REQUEST_NUMBER/comments" \
-              --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | contains(\"$marker\")) | .id"
-          } | tail -n 1)"
-          if [[ -n "$comment_id" ]]; then
-            gh api --method PATCH "repos/$GH_REPO/issues/comments/$comment_id" -f body="$body" >/dev/null
-          else
-            gh pr comment "$PULL_REQUEST_NUMBER" --body "$body"
+          if ! update_preview_comment_with_retry "🟢 Ready"; then
+            echo "::warning::Unable to mark the preview comment as ready."
           fi
+          preview_finished=true

.gitignore

@@ -3,6 +3,7 @@ node_modules/
 dist/
 .astro/
 .remote/
+.vercel-build/
 .temp/
 tmp/
 src/generated/

astro.config.ts

@@ -25,9 +25,8 @@ import { clickhouseSqlTransformer } from "./src/plugins/shiki-clickhouse-sql";
 // The site is served at clickhouse.com/docs behind the website Worker.
 export const BASE = "/docs";
 const buildScope = readScope();
-// Locale Workers share public images and Nimbus's static CSS through the
-// English Worker, but their compiled chunks must never collide. The website
-// router sends /docs/_astro-<locale>/* to the matching locale Worker.
+// Locale build shards share public images from the English output, but their
+// compiled chunks must never collide when all shards are merged.
 const buildAssetsDirectory = buildScope.locale === "en"
   ? "_astro"
   : `_astro-${buildScope.locale.toLowerCase()}`;
@@ -39,6 +38,10 @@ const remoteMounts = (JSON.parse(fs.readFileSync(new URL("./remotes.json", impor
 // pnpm install does not expose those transitive links to Rolldown, so use the
 // vendored ESM runtime instead of relying on node_modules symlink layout.
 const tslibModule = fileURLToPath(new URL("./src/shims/tslib.mjs", import.meta.url));
+const activeHomepageModule = fileURLToPath(new URL(
+  `./src/generated/homepage/${buildScope.locale.toLowerCase()}.jsx`,
+  import.meta.url,
+));
 
 function markdownSnippet(relativePath: string) {
   const source = fs.readFileSync(new URL(relativePath, import.meta.url), "utf8")
@@ -73,13 +76,16 @@ export default defineConfig({
   base: BASE,
   output: "static",
   build: { assets: buildAssetsDirectory },
-  publicDir: "./.remote/public-build",
-  // Overridable so parallel builds (CI shards, concurrent sessions) never
+  publicDir: process.env.DOCS_SKIP_PUBLIC === "1" ? "./.remote/public-empty" : "./.remote/public-build",
+  // Overridable so sequential locale shards and concurrent sessions never
   // share an output directory or the content-layer cache.
   outDir: process.env.DOCS_OUT_DIR ?? "./dist",
   cacheDir: process.env.DOCS_CACHE_DIR ?? "./node_modules/.astro",
   trailingSlash: "ignore",
   prefetch: { prefetchAll: true, defaultStrategy: "hover" },
+  experimental: {
+    incrementalBuild: true,
+  },
   markdown: {
     // Mermaid fences are rendered client-side (src/plugins/satteri-mermaid.ts).
     syntaxHighlight: { type: "shiki", excludeLangs: ["mermaid"] },
@@ -140,7 +146,7 @@ export default defineConfig({
     },
     plugins: [tailwindcss(), mintlifySnippets()],
     resolve: {
-      alias: { tslib: tslibModule },
+      alias: { tslib: tslibModule, "@active-homepage": activeHomepageModule },
       dedupe: ["react", "react-dom"],
     },
   },

bin/vercel-build.ts

@@ -7,6 +7,10 @@
  * post-processing run after every credential has been removed.
  */
 import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { readScope, type Locale } from "../src/lib/scope.ts";
+import { localeRouteName } from "../src/util/locales.ts";
 
 const root = process.cwd();
 const credentialVariables = [
@@ -44,6 +48,35 @@ function run(command: string, args: string[], environment: NodeJS.ProcessEnv): v
   execFileSync(command, args, { cwd: root, env: environment, stdio: "inherit" });
 }
 
+function shardEnvironment(
+  environment: NodeJS.ProcessEnv,
+  locale: "en" | Locale,
+  availableLocales: Locale[],
+  outDir: string,
+): NodeJS.ProcessEnv {
+  const child = { ...environment };
+  delete child.DOCS_LOCALES;
+  child.DOCS_BUILD_SHARD = "1";
+  child.DOCS_LOCALE = locale;
+  child.DOCS_AVAILABLE_LOCALES = availableLocales.join(",") || "none";
+  child.DOCS_EMIT_ENGLISH = locale === "en" ? "true" : "false";
+  child.DOCS_OUT_DIR = outDir;
+  child.DOCS_CACHE_DIR = path.join(root, "node_modules", ".astro", locale.toLowerCase());
+  if (locale === "en") delete child.DOCS_SKIP_PUBLIC;
+  else child.DOCS_SKIP_PUBLIC = "1";
+  return child;
+}
+
+function copyDirectory(source: string, destination: string, merge = false): void {
+  if (!fs.existsSync(source)) {
+    throw new Error(`vercel-build: expected shard output ${path.relative(root, source)} does not exist`);
+  }
+  if (!merge && fs.existsSync(destination)) {
+    throw new Error(`vercel-build: refusing to overwrite merged output ${path.relative(root, destination)}`);
+  }
+  fs.cpSync(source, destination, { recursive: true, force: merge, errorOnExist: !merge });
+}
+
 // The fetch child is the only process allowed to see the deployment OIDC
 // identity. It downloads bytes but never parses or imports remote-authored MDX.
 run(process.execPath, ["bin/fetch-remotes.ts"], { ...process.env });
@@ -54,5 +87,55 @@ const cleanEnvironment = sanitizedEnvironment();
 assertCredentialFree(cleanEnvironment);
 
 run("pnpm", ["run", "prepare:site"], cleanEnvironment);
-run("pnpm", ["exec", "astro", "build"], cleanEnvironment);
-run(process.execPath, ["bin/postbuild.ts"], cleanEnvironment);
+
+const scope = readScope(root);
+const finalOutDir = path.resolve(root, cleanEnvironment.DOCS_OUT_DIR ?? "dist");
+if (scope.locales.length === 0) {
+  const environment = shardEnvironment(cleanEnvironment, "en", [], finalOutDir);
+  run("pnpm", ["exec", "astro", "build"], environment);
+  run(process.execPath, ["bin/postbuild.ts"], environment);
+  process.exit(0);
+}
+
+// Each locale gets its own process, module graph, content collection, output,
+// and persistent Astro cache. Peak memory is bounded by one locale build while
+// the final deployment remains a single Vercel artifact.
+const shardsRoot = path.join(root, ".vercel-build", "shards");
+fs.rmSync(shardsRoot, { recursive: true, force: true });
+fs.mkdirSync(shardsRoot, { recursive: true });
+fs.mkdirSync(path.join(root, ".remote", "public-empty"), { recursive: true });
+
+const englishOutDir = path.join(shardsRoot, "en");
+run(
+  "pnpm",
+  ["exec", "astro", "build"],
+  shardEnvironment(cleanEnvironment, "en", scope.locales, englishOutDir),
+);
+
+const localeOutputs: Array<{ locale: Locale; outDir: string }> = [];
+for (const locale of scope.locales) {
+  const outDir = path.join(shardsRoot, locale.toLowerCase());
+  run(
+    "pnpm",
+    ["exec", "astro", "build"],
+    shardEnvironment(cleanEnvironment, locale, scope.locales, outDir),
+  );
+  localeOutputs.push({ locale, outDir });
+}
+
+fs.rmSync(finalOutDir, { recursive: true, force: true });
+copyDirectory(englishOutDir, finalOutDir);
+for (const { locale, outDir } of localeOutputs) {
+  copyDirectory(
+    path.join(outDir, localeRouteName(locale)),
+    path.join(finalOutDir, localeRouteName(locale)),
+    true,
+  );
+  copyDirectory(
+    path.join(outDir, `_astro-${locale.toLowerCase()}`),
+    path.join(finalOutDir, `_astro-${locale.toLowerCase()}`),
+  );
+}
+
+const postbuildEnvironment = { ...cleanEnvironment, DOCS_OUT_DIR: finalOutDir };
+run(process.execPath, ["bin/postbuild.ts"], postbuildEnvironment);

package.json

@@ -38,7 +38,7 @@
     "@readme/httpsnippet": "11.4.0",
     "@scalar/openapi-parser": "0.28.12",
     "@vercel/connect": "^2.0.2",
-    "astro": "~7.0.9",
+    "astro": "~7.3.2",
     "clsx": "^2.1.1",
     "mermaid": "^11.17.2",
     "openapi-sampler": "1.7.4",