perf(webapp,run-engine): cut CPU on the engine-facing worker-action routes

.gitignore

@@ -87,3 +87,6 @@ ailogger-output.log
 observability-map.json
 
 .claude/worktrees/
+
+# CPU benchmark artifacts (profiles + summaries)
+.bench/

.server-changes/reduce-webapp-cpu-on-worker-routes.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Cut webapp CPU usage by about a quarter on the routes that workers call most, freeing headroom at the same request rate. Detailed event-loop blocking traces are no longer recorded by default, because producing them was itself a large part of that cost.

apps/webapp/app/entry.server.tsx

@@ -18,7 +18,7 @@ import type { OperatingSystemPlatform } from "./components/primitives/OperatingS
 import { OperatingSystemContextProvider } from "./components/primitives/OperatingSystemProvider";
 import { assertRunOpsSplitSentinel, Prisma } from "./db.server";
 import { env } from "./env.server";
-import { eventLoopMonitor } from "./eventLoopMonitor.server";
+import { eventLoopMonitor, eventLoopUtilizationMonitor } from "./eventLoopMonitor.server";
 import { logger } from "./services/logger.server";
 import { buildImgSrcDirective, parseCspImageOrigins, withImgSrc } from "./utils/cspImageOrigins";
 import { singleton } from "./utils/singleton";
@@ -360,6 +360,10 @@ if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
   eventLoopMonitor.enable();
 }
 
+if (env.EVENT_LOOP_UTILIZATION_MONITOR_ENABLED === "1") {
+  eventLoopUtilizationMonitor.enable();
+}
+
 if (remoteBuildsEnabled()) {
   console.log("🏗️  Remote builds enabled");
 } else {

apps/webapp/app/env.server.ts

@@ -995,7 +995,8 @@ const EnvironmentSchema = z
 
     CENTS_PER_RUN: z.coerce.number().default(0),
 
-    EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"),
+    EVENT_LOOP_MONITOR_ENABLED: z.string().default("0"),
+    EVENT_LOOP_UTILIZATION_MONITOR_ENABLED: z.string().default("1"),
     MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
     MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
     MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(10_000),

apps/webapp/app/eventLoopMonitor.server.ts

@@ -89,25 +89,51 @@ function after(asyncId: number) {
   }
 }
 
+/**
+ * Per-async-resource blocked-loop detection. This is the expensive half: the
+ * hook fires for every async resource the process creates, and enabling any
+ * async hook also puts V8 on the slow path for promise instrumentation
+ * process-wide. On a request-heavy instance it costs roughly a seventh of all
+ * on-CPU time, which is why it is opt-in rather than on by default.
+ */
 export const eventLoopMonitor = singleton("eventLoopMonitor", () => {
   const hook = createHook({ init, before, after, destroy });
 
-  let stopEventLoopUtilizationMonitoring: () => void;
-
   return {
     enable: () => {
       console.log("🥸  Initializing event loop monitor");
 
       hook.enable();
-
-      stopEventLoopUtilizationMonitoring = startEventLoopUtilizationMonitoring();
     },
     disable: () => {
       console.log("🥸  Disabling event loop monitor");
 
       hook.disable();
+    },
+  };
+});
+
+/**
+ * The cheap half: a single interval timer reading `eventLoopUtilization()`.
+ * It costs nothing per request, so it stays on by default and is what a
+ * high-traffic instance should rely on when the async hook is too expensive.
+ */
+export const eventLoopUtilizationMonitor = singleton("eventLoopUtilizationMonitor", () => {
+  let stop: (() => void) | undefined;
 
-      stopEventLoopUtilizationMonitoring?.();
+  return {
+    enable: () => {
+      if (stop) {
+        return;
+      }
+
+      console.log("🥸  Initializing event loop utilization monitor");
+
+      stop = startEventLoopUtilizationMonitoring();
+    },
+    disable: () => {
+      stop?.();
+      stop = undefined;
     },
   };
 });

apps/webapp/package.json

@@ -25,7 +25,8 @@
     "upload:sourcemaps": "bash ./upload-sourcemaps.sh",
     "test": "vitest --no-file-parallelism",
     "test:perf": "vitest --config ./vitest.perf.config.ts --run",
-    "eval:dev": "evalite watch"
+    "eval:dev": "evalite watch",
+    "test:bench": "vitest --config ./vitest.bench.config.ts --run"
   },
   "dependencies": {
     "@ai-sdk/openai": "^3.0.0",

apps/webapp/test/bench/README.md

@@ -0,0 +1,91 @@
+# Engine CPU benchmarks
+
+Two benchmarks for the paths the production engine service spends its CPU in, plus a
+`.cpuprofile` analyzer. Neither runs in CI: they take minutes, attach the V8 profiler, and
+report numbers rather than assert on them.
+
+| bench | what it covers | where |
+| --- | --- | --- |
+| `engineHttp.bench.test.ts` | the full request stack for `engine/v1/worker-actions/*` | `apps/webapp` |
+| `runEngineLifecycle.bench.test.ts` | run-engine and run-queue with no HTTP in the way | `internal-packages/run-engine` |
+
+Artifacts (profiles + JSON summaries) land in `.bench/` at the repo root, which is gitignored.
+
+## HTTP bench
+
+Measures what a managed supervisor actually does: dequeue, start attempt, heartbeat,
+read latest snapshot, complete attempt. Needs a built webapp.
+
+```bash
+pnpm run build --filter webapp
+cd apps/webapp
+pnpm run test:bench
+```
+
+It spawns a real webapp against throwaway Postgres and Redis containers, seeds a production
+environment with a promoted managed deployment, fills the worker queue over the public
+trigger API, then drives a closed-loop supervisor pool for the measured window.
+
+The webapp is spawned with `--inspect` and profiled over CDP, so the profile covers only the
+measured window rather than boot. Event-loop utilization is sampled **inside** the webapp
+process over the same connection.
+
+Knobs:
+
+| var | default | meaning |
+| --- | --- | --- |
+| `BENCH_RUNS` | 1200 | runs queued before the window opens |
+| `BENCH_SUPERVISORS` | 16 | concurrent virtual supervisors |
+| `BENCH_HEARTBEATS` | 2 | heartbeats per run |
+| `BENCH_DURATION_MS` | 60000 | measured window |
+| `BENCH_SAMPLING_INTERVAL_US` | 200 | V8 sampling interval |
+| `BENCH_PROFILE_NAME` | `engine-http` | artifact basename |
+| `BENCH_EXTRA_ENV` | — | JSON merged into the webapp's env |
+| `BENCH_OUT_DIR` | `<repo>/.bench` | artifact directory |
+
+`BENCH_EXTRA_ENV` plus `BENCH_PROFILE_NAME` is how you A/B a single flag:
+
+```bash
+BENCH_RUNS=5000 BENCH_SUPERVISORS=24 BENCH_DURATION_MS=90000 \
+  BENCH_PROFILE_NAME=engine-http-no-elm \
+  BENCH_EXTRA_ENV='{"EVENT_LOOP_MONITOR_ENABLED":"0"}' \
+  pnpm run test:bench
+```
+
+Run the same size for both arms and compare `on-cpu ms per completed run` rather than
+throughput: throughput on a laptop moves ~5% run to run, on-CPU per unit of work is far
+steadier.
+
+## Run-engine bench
+
+No HTTP, no webapp: drives `RunEngine` directly so engine and queue costs are not mixed with
+request-stack overhead. Profiles two phases separately, because blending them hides which one
+owns a hot frame.
+
+```bash
+cd internal-packages/run-engine
+pnpm run test:bench
+```
+
+Knobs: `BENCH_RUNS`, `BENCH_CONSUMERS`, `BENCH_HEARTBEATS`, `BENCH_CONCURRENCY_LIMIT`,
+`BENCH_SAMPLING_INTERVAL_US`, `BENCH_OUT_DIR`.
+
+The driver shares a process with the code under measurement, so its own cost is in the
+profile. It is a thin await loop and appears under its own frames rather than smeared across
+engine frames.
+
+## Analyzing a profile
+
+```bash
+pnpm --filter webapp exec tsx test/bench/analyzeProfile.ts .bench/engine-http.cpuprofile --top 30
+```
+
+Three views: CPU by bucket (which package owns the cycles), hottest frames by self time (what
+to go fix), and hottest frames by total time (entry points, and a check that the load
+exercised the route mix you intended). Frames are symbolicated through the build's source
+maps, so bundled chunks report as the source files they came from.
+
+Percentages are shares of **on-CPU** time, with V8's `(idle)` and `(program)` excluded. A
+share of wall clock would make everything look cheap whenever the bench was IO-bound.
+
+`--json <path>` writes the full analysis for diffing two runs.

apps/webapp/test/bench/analyzeProfile.ts

@@ -0,0 +1,66 @@
+#!/usr/bin/env tsx
+/**
+ * Ranks where a `.cpuprofile` spent its cycles.
+ *
+ *   pnpm --filter webapp exec tsx test/bench/analyzeProfile.ts <profile> [--top 40] [--json out.json]
+ *
+ * `--root` overrides the repo root used to make source paths relative and to
+ * find the build's source maps; it defaults to the repo containing this file.
+ */
+import { readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { analyzeProfile, formatAnalysis, type CpuProfile } from "./lib/profileAnalysis";
+
+function parseArgs(argv: string[]): {
+  profilePath?: string;
+  top: number;
+  json?: string;
+  root: string;
+} {
+  const here = typeof __dirname === "string" ? __dirname : import.meta.dirname;
+
+  const defaults = {
+    top: 30,
+    root: resolve(here, "..", "..", "..", ".."),
+  };
+
+  let profilePath: string | undefined;
+  let top = defaults.top;
+  let json: string | undefined;
+  let root = defaults.root;
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]!;
+    if (arg === "--top") {
+      const raw = argv[++i];
+      const parsed = Number(raw);
+      if (!Number.isFinite(parsed) || parsed <= 0) {
+        console.error(`--top expects a positive number, got "${raw ?? ""}"`);
+        process.exit(1);
+      }
+      top = parsed;
+    } else if (arg === "--json") json = argv[++i];
+    else if (arg === "--root") root = resolve(argv[++i]!);
+    else if (!arg.startsWith("--")) profilePath = arg;
+  }
+
+  return { profilePath, top, json, root };
+}
+
+const { profilePath, top, json, root } = parseArgs(process.argv.slice(2));
+
+if (!profilePath) {
+  console.error("usage: analyzeProfile.ts <path-to-.cpuprofile> [--top N] [--json out.json]");
+  process.exit(1);
+}
+
+const profile = JSON.parse(readFileSync(profilePath, "utf8")) as CpuProfile;
+const analysis = analyzeProfile(profile, root);
+
+console.log(`\n=== ${profilePath} ===`);
+console.log(formatAnalysis(analysis, top));
+
+if (json) {
+  writeFileSync(json, JSON.stringify(analysis, null, 2));
+  console.log(`\nwrote ${json}`);
+}