From dc20f17ffbd4184da1850d1fab0f61fc9d1c14f1 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Sat, 1 Aug 2026 07:55:06 +0100 Subject: [PATCH 1/2] perf(webapp,run-engine,database): resolve the newest worker and deployment by createdAt Ordering these lookups by id makes Postgres scan the primary key backwards betting on an early match for the target environment. When that bet loses the scan crosses a large part of the index. Ordering by createdAt then id keeps the lookup on a composite index instead. Adds (environmentId, createdAt) on WorkerDeployment, which had no matching index. BackgroundWorker already had one. --- .../worker-deployment-lookup-ordering.md | 6 + .../clickhouseRunsRepository.server.ts | 3 - .../app/v3/models/workerDeployment.server.ts | 4 +- .../controlPlaneResolver.server.ts | 5 +- .../controlPlaneResolver.server.test.ts | 111 +++++++++++++- .../migration.sql | 1 + .../database/prisma/schema.prisma | 1 + .../src/engine/controlPlaneResolver.ts | 11 +- ...dequeueSystem.controlPlaneResolver.test.ts | 139 ++++++++++++++++++ 9 files changed, 261 insertions(+), 20 deletions(-) create mode 100644 .server-changes/worker-deployment-lookup-ordering.md create mode 100644 internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql diff --git a/.server-changes/worker-deployment-lookup-ordering.md b/.server-changes/worker-deployment-lookup-ordering.md new file mode 100644 index 00000000000..1dbfbdd253a --- /dev/null +++ b/.server-changes/worker-deployment-lookup-ordering.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Speeds up resolving the latest worker version and deployment for an environment, removing an occasional stall when triggering runs in projects that have accumulated many deployed versions. diff --git a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts index 2aa9d5f3a68..c9fefd1da10 100644 --- a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts @@ -271,9 +271,6 @@ export class ClickHouseRunsRepository implements IRunsRepository { in: ids, }, }, - orderBy: { - id: "desc", - }, select: { id: true, friendlyId: true, diff --git a/apps/webapp/app/v3/models/workerDeployment.server.ts b/apps/webapp/app/v3/models/workerDeployment.server.ts index 5934cd07620..62d7ea63130 100644 --- a/apps/webapp/app/v3/models/workerDeployment.server.ts +++ b/apps/webapp/app/v3/models/workerDeployment.server.ts @@ -133,9 +133,7 @@ export async function findCurrentWorkerDeployment({ environmentId, type, }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], select: { id: true, imageReference: true, diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts index d6a706c9665..eb19a7fb6c1 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts @@ -432,7 +432,7 @@ export class ControlPlaneResolver { // MANAGED deployment. const latestV2Deployment = await client.workerDeployment.findFirst({ where: { environmentId, type: "MANAGED" }, - orderBy: { id: "desc" }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], include: { worker: { include: { tasks: true, queues: true } } }, }); @@ -455,7 +455,6 @@ export class ControlPlaneResolver { const worker = await client.backgroundWorker.findFirst({ where: { id: workerId }, include: { deployment: true, tasks: true, queues: true }, - orderBy: { id: "desc" }, }); if (!worker) { @@ -472,7 +471,7 @@ export class ControlPlaneResolver { const worker = await client.backgroundWorker.findFirst({ where: { runtimeEnvironmentId: environmentId }, include: { tasks: true, queues: true }, - orderBy: { id: "desc" }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], }); if (!worker) { diff --git a/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts b/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts index e85843114db..af734ca1f75 100644 --- a/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts +++ b/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts @@ -62,7 +62,12 @@ async function seedControlPlane(prisma: PrismaClient) { async function seedWorker( prisma: PrismaClient, ctx: { projectId: string; environmentId: string }, - opts?: { promote?: boolean } + opts?: { + promote?: boolean; + createdAt?: Date; + deploymentType?: "MANAGED" | "UNMANAGED" | "V1"; + deploymentCreatedAt?: Date; + } ) { const n = seedCounter++; const worker = await prisma.backgroundWorker.create({ @@ -74,6 +79,7 @@ async function seedWorker( version: `2024.1.${n}`, metadata: {}, engine: "V2", + ...(opts?.createdAt ? { createdAt: opts.createdAt } : {}), }, }); const task = await prisma.backgroundWorkerTask.create({ @@ -104,11 +110,12 @@ async function seedWorker( contentHash: `hash_${n}`, version: worker.version, shortCode: `dep_${n}`, - type: "MANAGED", + type: opts?.deploymentType ?? "MANAGED", status: "DEPLOYED", projectId: ctx.projectId, environmentId: ctx.environmentId, workerId: worker.id, + ...(opts?.deploymentCreatedAt ? { createdAt: opts.deploymentCreatedAt } : {}), }, }); await prisma.workerDeploymentPromotion.create({ @@ -748,3 +755,103 @@ heteroPostgresTest( expect(reads()).toBe(readsAfterFirst * 2); } ); + +heteroPostgresTest( + "resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id", + async ({ prisma14 }) => { + const { environment, project } = await seedControlPlane(prisma14); + const ctx = { projectId: project.id, environmentId: environment.id }; + + const newest = await seedWorker(prisma14, ctx, { + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }); + const oldest = await seedWorker(prisma14, ctx, { + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }); + + expect(oldest.worker.id > newest.worker.id).toBe(true); + expect(oldest.worker.createdAt < newest.worker.createdAt).toBe(true); + + const resolver = new ControlPlaneResolver({ + controlPlaneReplica: prisma14, + controlPlanePrimary: prisma14, + cache: new ControlPlaneCache(), + splitEnabled: () => true, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: environment.id, + type: "DEVELOPMENT", + }); + + expect(resolved).not.toBeNull(); + expect(resolved!.worker.id).toBe(newest.worker.id); + } +); + +heteroPostgresTest( + "resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id", + async ({ prisma14 }) => { + const { environment, project } = await seedControlPlane(prisma14); + const ctx = { projectId: project.id, environmentId: environment.id }; + + await seedWorker(prisma14, ctx, { promote: true, deploymentType: "V1" }); + + const newest = await seedWorker(prisma14, ctx, { + promote: false, + deploymentCreatedAt: new Date("2026-07-31T12:00:00.000Z"), + }); + const oldest = await seedWorker(prisma14, ctx, { + promote: false, + deploymentCreatedAt: new Date("2026-07-30T12:00:00.000Z"), + }); + + const newestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: `deployment_newest_${environment.id}`, + contentHash: "hash_newest", + version: newest.worker.version, + shortCode: "dep_newest", + type: "MANAGED", + status: "DEPLOYED", + projectId: project.id, + environmentId: environment.id, + workerId: newest.worker.id, + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: `deployment_oldest_${environment.id}`, + contentHash: "hash_oldest", + version: oldest.worker.version, + shortCode: "dep_oldest", + type: "MANAGED", + status: "DEPLOYED", + projectId: project.id, + environmentId: environment.id, + workerId: oldest.worker.id, + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldestDeployment.id > newestDeployment.id).toBe(true); + expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true); + + const resolver = new ControlPlaneResolver({ + controlPlaneReplica: prisma14, + controlPlanePrimary: prisma14, + cache: new ControlPlaneCache(), + splitEnabled: () => true, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: environment.id, + type: "PRODUCTION", + }); + + expect(resolved).not.toBeNull(); + expect(resolved!.deployment!.id).toBe(newestDeployment.id); + expect(resolved!.worker.id).toBe(newest.worker.id); + } +); diff --git a/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql b/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql new file mode 100644 index 00000000000..57d9eec96c4 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "WorkerDeployment_environmentId_createdAt_idx" ON "public"."WorkerDeployment"("environmentId", "createdAt"); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 74c6d72a9c5..ca1d868ab04 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2208,6 +2208,7 @@ model WorkerDeployment { @@unique([projectId, shortCode]) @@unique([environmentId, version]) @@index([commitSHA]) + @@index([environmentId, createdAt]) } enum WorkerDeploymentStatus { diff --git a/internal-packages/run-engine/src/engine/controlPlaneResolver.ts b/internal-packages/run-engine/src/engine/controlPlaneResolver.ts index 89b0e15c48c..da5f7b00603 100644 --- a/internal-packages/run-engine/src/engine/controlPlaneResolver.ts +++ b/internal-packages/run-engine/src/engine/controlPlaneResolver.ts @@ -234,9 +234,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { tasks: true, queues: true, }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], }); if (!worker) { @@ -256,9 +254,6 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { tasks: true, queues: true, }, - orderBy: { - id: "desc", - }, }); if (!worker) { @@ -315,9 +310,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { environmentId, type: "MANAGED", }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], include: { worker: { include: { diff --git a/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts b/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts index 5ede5c89db2..3ea56bde8a6 100644 --- a/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts +++ b/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts @@ -538,3 +538,142 @@ describe("DequeueSystem controlPlaneResolver (single-DB passthrough)", () => { } ); }); + +heteroPostgresTest( + "resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id", + async ({ prisma14 }) => { + const cp = await seedControlPlane( + prisma14 as unknown as PrismaClient, + "cpord", + "ordering-task" + ); + + await prisma14.backgroundWorker.update({ + where: { id: cp.worker.id }, + data: { createdAt: new Date("2026-07-01T12:00:00.000Z") }, + }); + + const newest = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_newest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260731.1", + metadata: {}, + engine: "V2", + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldest = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_oldest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260730.1", + metadata: {}, + engine: "V2", + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldest.id > newest.id).toBe(true); + expect(oldest.createdAt < newest.createdAt).toBe(true); + + const resolver = new PassthroughControlPlaneResolver({ + prisma: prisma14 as unknown as PrismaClient, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: cp.environment.id, + type: "DEVELOPMENT", + }); + + assertNonNullable(resolved); + expect(resolved.worker.id).toBe(newest.id); + } +); + +heteroPostgresTest( + "resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id", + async ({ prisma14 }) => { + const cp = await seedControlPlane( + prisma14 as unknown as PrismaClient, + "cpfall", + "fallback-task" + ); + + await prisma14.workerDeployment.update({ + where: { id: cp.deployment.id }, + data: { type: "V1" }, + }); + + const newestWorker = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_newest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260731.1", + metadata: {}, + engine: "V2", + }, + }); + const oldestWorker = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_oldest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260730.1", + metadata: {}, + engine: "V2", + }, + }); + + const newestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: "hash_newest", + version: "20260731.1", + shortCode: "short_code_newest", + status: "DEPLOYED", + projectId: cp.project.id, + environmentId: cp.environment.id, + workerId: newestWorker.id, + type: "MANAGED", + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: "hash_oldest", + version: "20260730.1", + shortCode: "short_code_oldest", + status: "DEPLOYED", + projectId: cp.project.id, + environmentId: cp.environment.id, + workerId: oldestWorker.id, + type: "MANAGED", + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldestDeployment.id > newestDeployment.id).toBe(true); + expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true); + + const resolver = new PassthroughControlPlaneResolver({ + prisma: prisma14 as unknown as PrismaClient, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: cp.environment.id, + type: "PRODUCTION", + }); + + assertNonNullable(resolved); + expect(resolved.deployment?.id).toBe(newestDeployment.id); + } +); From e93a16dd9b11bfe857f1ce7fb3da33a2af044a83 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Sat, 1 Aug 2026 09:15:36 +0100 Subject: [PATCH 2/2] fix(webapp): honor the caller's prisma client in the latest-deployment fallback findCurrentWorkerDeployment resolved the caller-supplied client for the promotion read but used the module-level client for the fallback read, so callers passing a replica silently read the primary for that one query. --- apps/webapp/app/v3/models/workerDeployment.server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/webapp/app/v3/models/workerDeployment.server.ts b/apps/webapp/app/v3/models/workerDeployment.server.ts index 62d7ea63130..48995bf2b29 100644 --- a/apps/webapp/app/v3/models/workerDeployment.server.ts +++ b/apps/webapp/app/v3/models/workerDeployment.server.ts @@ -128,7 +128,7 @@ export async function findCurrentWorkerDeployment({ } // We need to get the latest deployment of the given type - const latestDeployment = await prisma.workerDeployment.findFirst({ + const latestDeployment = await $prisma.workerDeployment.findFirst({ where: { environmentId, type,