From d760741d6793d49210c3ca3ad3c143370b16c20f Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 15:50:36 +0100 Subject: [PATCH 01/15] perf(ci): rebalance webapp test shards --- .github/workflows/e2e-webapp.yml | 58 +- .github/workflows/unit-tests-webapp.yml | 67 +- test-timings.json | 784 ++++++++++++++++-------- 3 files changed, 611 insertions(+), 298 deletions(-) diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml index a05b4525d12..ad7abea0383 100644 --- a/.github/workflows/e2e-webapp.yml +++ b/.github/workflows/e2e-webapp.yml @@ -57,7 +57,7 @@ jobs: version: 10.33.2 - name: โŽ” Setup node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 with: node-version: 24.18.0 cache: "pnpm" @@ -73,18 +73,52 @@ jobs: if: ${{ !env.DOCKERHUB_USERNAME }} run: echo "DockerHub login skipped because secrets are not available." - - name: ๐Ÿณ Pre-pull testcontainer images + - name: ๐Ÿ“ฅ Prepare deps and testcontainer images run: | - echo "Pre-pulling Docker images with authenticated session..." - docker pull postgres:14 - docker pull redis:7.2 - docker pull testcontainers/ryuk:0.14.0 - docker pull ghcr.io/s2-streamstore/s2:0.40.0@sha256:b26249e2ede0949755f5af8028185dc2bcfc3aa2db21eb9610543d144eb6ee9d - docker pull minio/minio:latest - echo "Image pre-pull complete" - - - name: ๐Ÿ“ฅ Download deps - run: pnpm install --frozen-lockfile + # Pull images concurrently with dependency installation. Retry each pull because + # registry timeouts are a recurring transient CI flake. + pull() { + for attempt in 1 2 3; do + docker pull "$1" && return 0 + echo "::warning::docker pull $1 failed (attempt ${attempt}/3); retrying in 10s" + sleep 10 + done + echo "::error::docker pull $1 failed after 3 attempts" + return 1 + } + + pull_images() { + local pids=() + local failed=0 + for image in \ + postgres:14 \ + redis:7.2 \ + testcontainers/ryuk:0.14.0 \ + ghcr.io/s2-streamstore/s2:0.40.0@sha256:b26249e2ede0949755f5af8028185dc2bcfc3aa2db21eb9610543d144eb6ee9d \ + minio/minio:latest + do + pull "$image" & + pids+=("$!") + done + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + failed=1 + fi + done + return "$failed" + } + + echo "Installing dependencies and pre-pulling Docker images..." + pull_images & + pull_pid=$! + install_status=0 + pnpm install --frozen-lockfile || install_status=$? + pull_status=0 + wait "$pull_pid" || pull_status=$? + if (( install_status != 0 || pull_status != 0 )); then + exit 1 + fi + echo "Dependency install and image pre-pull complete" - name: ๐Ÿ“€ Generate Prisma Client run: pnpm run generate diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index bec77bbbc4b..4db1302bdaa 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -14,18 +14,17 @@ on: jobs: unitTests: name: "๐Ÿงช Unit Tests: Webapp" - # 10 shards on 16x machines: webapp test throughput is limited per-machine (one - # docker daemon + disk absorbing all the per-file Postgres/ClickHouse container - # spin-up), so many machines beats few big ones - fewer/bigger (3x32) measured - # SLOWER than 10x8. The 16x (vs 8x) gives the fork pool the CPU headroom the 8x - # runners lacked. Setup overhead per machine is ~1 min on warm runners. + # Webapp test throughput is limited per-machine (one docker daemon + disk absorbing + # all the per-file Postgres/ClickHouse container spin-up), so many machines beats + # few big ones - fewer/bigger (3x32) measured slower than 10x8. The 16x (vs 8x) + # gives the fork pool the CPU headroom the 8x runners lacked. runs-on: warp-ubuntu-latest-x64-16x strategy: # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard fail-fast: false matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] - shardTotal: [12] + shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + shardTotal: [24] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SHARD_INDEX: ${{ matrix.shardIndex }} @@ -69,7 +68,7 @@ jobs: version: 10.33.2 - name: โŽ” Setup node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 with: node-version: 24.18.0 cache: "pnpm" @@ -85,9 +84,10 @@ jobs: if: ${{ !env.DOCKERHUB_USERNAME }} run: echo "DockerHub login skipped because secrets are not available." - - name: ๐Ÿณ Pre-pull testcontainer images + - name: ๐Ÿ“ฅ Prepare deps and testcontainer images run: | - # Retry each pull - DockerHub registry timeouts are a recurring transient CI flake. + # Pull images concurrently with dependency installation. Retry each pull because + # DockerHub registry timeouts are a recurring transient CI flake. pull() { for attempt in 1 2 3; do docker pull "$1" && return 0 @@ -97,18 +97,41 @@ jobs: echo "::error::docker pull $1 failed after 3 attempts" return 1 } - echo "Pre-pulling Docker images with authenticated session..." - pull postgres:14 - pull postgres:17 - pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251 - pull redis:7.2 - pull testcontainers/ryuk:0.14.0 - pull electricsql/electric:1.2.4@sha256:20da3d0b0e74926c5623392db67fd56698b9e374c4aeb6cb5cadeb8fea171c36 - pull minio/minio:latest - echo "Image pre-pull complete" - - - name: ๐Ÿ“ฅ Download deps - run: pnpm install --frozen-lockfile + + pull_images() { + local pids=() + local failed=0 + for image in \ + postgres:14 \ + postgres:17 \ + clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251 \ + redis:7.2 \ + testcontainers/ryuk:0.14.0 \ + electricsql/electric:1.2.4@sha256:20da3d0b0e74926c5623392db67fd56698b9e374c4aeb6cb5cadeb8fea171c36 \ + minio/minio:latest + do + pull "$image" & + pids+=("$!") + done + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + failed=1 + fi + done + return "$failed" + } + + echo "Installing dependencies and pre-pulling Docker images..." + pull_images & + pull_pid=$! + install_status=0 + pnpm install --frozen-lockfile || install_status=$? + pull_status=0 + wait "$pull_pid" || pull_status=$? + if (( install_status != 0 || pull_status != 0 )); then + exit 1 + fi + echo "Dependency install and image pre-pull complete" - name: ๐Ÿ“€ Generate Prisma Client run: pnpm run generate diff --git a/test-timings.json b/test-timings.json index 33fa9c5f32b..0dca9f10562 100644 --- a/test-timings.json +++ b/test-timings.json @@ -1,269 +1,525 @@ { - "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 26, - "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 2091, - "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 49515, - "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 132479, - "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 132404, - "apps/webapp/app/utils/friendlyId.test.ts": 59, - "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 25, - "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 3372, - "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 36, - "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 477, - "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 27, - "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 6305, - "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 24, - "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 546, - "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 511, - "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 6073, - "apps/webapp/app/v3/runStore.server.test.ts": 8480, - "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 5922, - "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 83, - "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 4096, - "apps/webapp/test/GCRARateLimiter.test.ts": 4787, - "apps/webapp/test/SpanPresenter.readthrough.test.ts": 9668, - "apps/webapp/test/activitySeries.server.test.ts": 25, - "apps/webapp/test/aiTitleRateLimiter.test.ts": 744, - "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 97711, - "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 179054, - "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 5727, - "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 5378, - "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 11836, - "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 3476, - "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 8669, - "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 12490, - "apps/webapp/test/apiRunListPresenter.test.ts": 148237, - "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 6843, - "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 3321, - "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 9115, - "apps/webapp/test/batchListPresenter.readroute.test.ts": 11138, - "apps/webapp/test/batchPresenter.test.ts": 16534, - "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 1959, - "apps/webapp/test/batchRunAccess.test.ts": 8353, - "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 6556, - "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 1037, - "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 5624, - "apps/webapp/test/billingAlertsFormat.test.ts": 34, - "apps/webapp/test/billingLimit.schemas.test.ts": 40, - "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 16155, - "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 3424, - "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 21, - "apps/webapp/test/billingLimitConvergeResolve.test.ts": 200, - "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 609, - "apps/webapp/test/billingLimitHit.test.ts": 25, - "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 35, - "apps/webapp/test/billingLimitQueuedRuns.test.ts": 12281, - "apps/webapp/test/billingLimitReconcileTick.test.ts": 213, - "apps/webapp/test/billingLimitReconciliation.test.ts": 630, - "apps/webapp/test/billingLimitResolve.test.ts": 17, - "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 26, - "apps/webapp/test/billingLimitsRoute.test.ts": 1102, - "apps/webapp/test/branchableEnvironment.test.ts": 20, - "apps/webapp/test/bufferedTriggerPayload.test.ts": 25, - "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 6183, + "apps/webapp/app/components/code/StreamdownRenderer.test.ts": 284, + "apps/webapp/app/components/code/tsql/tsqlLinter.test.ts": 119, + "apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.render.test.ts": 21, + "apps/webapp/app/components/dashboard-agent/InvestigationCard.render.test.ts": 14, + "apps/webapp/app/components/dashboard-agent/InvestigationCard.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/ReportView.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/WatchChips.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/agent-shortcuts.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/ai-entry-points.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/ask-ai-channels.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/askAiOpenRequest.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/chat-layout.test.ts": 7, + "apps/webapp/app/components/dashboard-agent/coalesced-reload.test.ts": 7, + "apps/webapp/app/components/dashboard-agent/composer-escape.test.ts": 1, + "apps/webapp/app/components/dashboard-agent/dashboardAgentOpenRequest.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/demo/demo.test.ts": 21, + "apps/webapp/app/components/dashboard-agent/diagnosis-actions.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/explicit-prompt.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/header-labels.test.ts": 1, + "apps/webapp/app/components/dashboard-agent/investigate-prompts.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/investigation-winners.test.ts": 9, + "apps/webapp/app/components/dashboard-agent/last-chat-storage.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/message-limits.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/message-order.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/message-quota.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/model-markdown.test.ts": 142, + "apps/webapp/app/components/dashboard-agent/navigate-target.test.ts": 7, + "apps/webapp/app/components/dashboard-agent/opened-chat.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/page-label.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/panel-escape.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/pending-intents.test.ts": 7, + "apps/webapp/app/components/dashboard-agent/pending-turn.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/progress-line.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/report-block-adapter.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/report-spark.test.ts": 97, + "apps/webapp/app/components/dashboard-agent/resolve-uris.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/retry-action.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/run-id.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/send-request.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/settled-transcript.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts": 19, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/promoted.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts": 26, + "apps/webapp/app/components/dashboard-agent/thinking-marker.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/tool-labels.test.ts": 1, + "apps/webapp/app/components/dashboard-agent/tooltip-accessible-name.test.ts": 1, + "apps/webapp/app/components/dashboard-agent/turn-error.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/turn-navigation.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/turn-teardown.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/unread-counts.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/unread-work.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/view-actions.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/view-blocks.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/view-catalog.test.ts": 7, + "apps/webapp/app/components/dashboard-agent/wake-banner.test.ts": 11, + "apps/webapp/app/components/dashboard-agent/wake-poll.test.ts": 11, + "apps/webapp/app/components/dashboard-agent/watch-activity.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/watch-card.test.ts": 33, + "apps/webapp/app/components/dashboard-agent/watch-chips.test.ts": 3, + "apps/webapp/app/components/queues/queue-name.test.ts": 1, + "apps/webapp/app/components/queues/queue-thresholds.test.ts": 2, + "apps/webapp/app/presenters/v3/reports/report-layout.test.ts": 23, + "apps/webapp/app/routes/storybook.agent-ui/fixtures.test.ts": 6, + "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 3, + "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 2, + "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 796, + "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 2972, + "apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts": 2896, + "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 2743, + "apps/webapp/app/utils/apiKeys.test.ts": 6, + "apps/webapp/app/utils/boundedRequestBody.server.test.ts": 9, + "apps/webapp/app/utils/cspImageOrigins.test.ts": 3, + "apps/webapp/app/utils/databaseMetrics.server.test.ts": 1, + "apps/webapp/app/utils/deeplinkPages.test.ts": 12, + "apps/webapp/app/utils/environmentAccess.test.ts": 2, + "apps/webapp/app/utils/friendlyId.test.ts": 6, + "apps/webapp/app/utils/impersonationPaths.test.ts": 5, + "apps/webapp/app/utils/impersonationState.test.ts": 2, + "apps/webapp/app/utils/localHostGuard.test.ts": 2, + "apps/webapp/app/utils/logSearch.test.ts": 5, + "apps/webapp/app/utils/nullBytes.test.ts": 1, + "apps/webapp/app/utils/pageSwitching.test.ts": 19, + "apps/webapp/app/utils/pageTitle.test.ts": 4, + "apps/webapp/app/utils/plainCustomerCards.test.ts": 6, + "apps/webapp/app/utils/prismaConnectionUrl.test.ts": 1, + "apps/webapp/app/utils/requestIdempotency.test.ts": 1, + "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 4, + "apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts": 6, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.dispatchFreshness.test.ts": 6423, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 4780, + "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 2, + "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 5, + "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 13, + "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 12533, + "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 1, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 10, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 5, + "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 15874, + "apps/webapp/app/v3/runStore.server.test.ts": 13921, + "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 22710, + "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 3, + "apps/webapp/app/v3/utils/priority.test.ts": 2, + "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 5039, + "apps/webapp/test/GCRARateLimiter.test.ts": 4477, + "apps/webapp/test/SpanPresenter.readthrough.test.ts": 11106, + "apps/webapp/test/activitySeries.server.test.ts": 6, + "apps/webapp/test/additionalApiKeyIssuance.test.ts": 3, + "apps/webapp/test/aiTitleRateLimiter.test.ts": 55, + "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 8260, + "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 10265, + "apps/webapp/test/apiAuthActorClaim.test.ts": 4, + "apps/webapp/test/apiAuthScope.test.ts": 7, + "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 16921, + "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 15142, + "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 18108, + "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 1130, + "apps/webapp/test/apiBuilderAuthorization.test.ts": 1, + "apps/webapp/test/apiKeysPresenter.test.ts": 4705, + "apps/webapp/test/apiRateLimitJwtActor.test.ts": 4, + "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 946, + "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 9845, + "apps/webapp/test/apiRunListPresenter.test.ts": 74771, + "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 18402, + "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 1713, + "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 21866, + "apps/webapp/test/authFeatureControls.test.ts": 3, + "apps/webapp/test/authorizationCodeConsent.test.ts": 4459, + "apps/webapp/test/authorizationRateLimitMiddleware.test.ts": 1, + "apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts": 107, + "apps/webapp/test/batchListPresenter.readroute.test.ts": 26765, + "apps/webapp/test/batchPresenter.test.ts": 20446, + "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 6, + "apps/webapp/test/batchRunAccess.test.ts": 4049, + "apps/webapp/test/batchServices.replicaLag.test.ts": 20440, + "apps/webapp/test/batchStreamGrants.test.ts": 199, + "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 8680, + "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 5, + "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 14259, + "apps/webapp/test/billingAlertsDefaults.test.ts": 2, + "apps/webapp/test/billingAlertsFormat.test.ts": 2, + "apps/webapp/test/billingLimit.schemas.test.ts": 7, + "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 13473, + "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 6177, + "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 2, + "apps/webapp/test/billingLimitConvergeResolve.test.ts": 15, + "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 5, + "apps/webapp/test/billingLimitHit.test.ts": 3, + "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 2, + "apps/webapp/test/billingLimitQueuedRuns.test.ts": 29233, + "apps/webapp/test/billingLimitReconcileTick.test.ts": 7, + "apps/webapp/test/billingLimitReconciliation.test.ts": 13124, + "apps/webapp/test/billingLimitResolve.test.ts": 2, + "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 2, + "apps/webapp/test/billingLimitsRoute.test.ts": 16, + "apps/webapp/test/branchableEnvironment.test.ts": 3, + "apps/webapp/test/bufferedTriggerPayload.test.ts": 4, + "apps/webapp/test/bulkActionV2.replicaLag.test.ts": 8967, + "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 15978, "apps/webapp/test/calculateNextSchedule.test.ts": 208, - "apps/webapp/test/chartActivityTimeAxis.test.ts": 29, - "apps/webapp/test/chartXAxisTicks.test.ts": 29, - "apps/webapp/test/chartZoomRange.test.ts": 22, - "apps/webapp/test/chat-snapshot-integration.test.ts": 1896, - "apps/webapp/test/checkPermissions.test.ts": 22, - "apps/webapp/test/clickhouseFactory.test.ts": 3978, - "apps/webapp/test/components/DateTime.test.ts": 784, - "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 136, - "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 273, - "apps/webapp/test/components/runs/v3/RunTag.test.ts": 190, - "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 339, - "apps/webapp/test/computeBucket.test.ts": 123, - "apps/webapp/test/computeMigration.test.ts": 22, - "apps/webapp/test/concurrentFlushScheduler.test.ts": 716, - "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 8603, - "apps/webapp/test/crossSeamGuard.proof.test.ts": 5538, - "apps/webapp/test/dependentAttemptScope.test.ts": 17, - "apps/webapp/test/detectQueryTables.test.ts": 259, - "apps/webapp/test/detectbadJsonStrings.test.ts": 73, - "apps/webapp/test/devBranchServices.test.ts": 4029, - "apps/webapp/test/devPresenceRecency.test.ts": 1179, - "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2678, - "apps/webapp/test/duplicateTaskIds.test.ts": 20, - "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1730, - "apps/webapp/test/emailPattern.test.ts": 18, - "apps/webapp/test/engine/batchPayloads.test.ts": 5365, - "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 10574, - "apps/webapp/test/engine/streamBatchItems.test.ts": 19344, - "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 4014, - "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 133741, - "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 91796, - "apps/webapp/test/engine/triggerTask.debounce.test.ts": 134153, - "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133523, - "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 173067, - "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 173296, - "apps/webapp/test/engine/triggerTask.residency.test.ts": 172996, - "apps/webapp/test/engine/triggerTask.test.ts": 92296, - "apps/webapp/test/environmentSort.test.ts": 29, - "apps/webapp/test/environmentVariableDeduplication.test.ts": 24, - "apps/webapp/test/environmentVariableRules.test.ts": 19, - "apps/webapp/test/environmentVariablesEnvironments.test.ts": 3690, - "apps/webapp/test/environmentVariablesRepository.test.ts": 4145, - "apps/webapp/test/errorFingerprinting.test.ts": 28, - "apps/webapp/test/errorGroupWebhook.test.ts": 59, - "apps/webapp/test/findEnvironmentByApiKey.test.ts": 3764, - "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 5555, - "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 8848, - "apps/webapp/test/getDeploymentImageRef.test.ts": 306, - "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 6408, - "apps/webapp/test/googleEmailVerification.test.ts": 27, - "apps/webapp/test/httpErrors.test.ts": 39, - "apps/webapp/test/idempotencyDedupResidency.test.ts": 9623, - "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 5778, - "apps/webapp/test/inviteRoleLadder.test.ts": 18, - "apps/webapp/test/member.server.test.ts": 5194, - "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 274, - "apps/webapp/test/mfaRateLimiter.test.ts": 632, - "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 850, - "apps/webapp/test/mollifierClaimResolution.test.ts": 469, - "apps/webapp/test/mollifierDecisionLabels.test.ts": 47, - "apps/webapp/test/mollifierDrainerHandler.test.ts": 450, - "apps/webapp/test/mollifierDrainerWorker.test.ts": 2020, - "apps/webapp/test/mollifierDrainingGauge.test.ts": 716, - "apps/webapp/test/mollifierGate.test.ts": 492, - "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 423, - "apps/webapp/test/mollifierMollify.test.ts": 223, - "apps/webapp/test/mollifierMutateWithFallback.test.ts": 445, - "apps/webapp/test/mollifierReadFallback.test.ts": 405, - "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 249, - "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 522, - "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 448, - "apps/webapp/test/mollifierStaleSweep.test.ts": 1047, - "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 559, - "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 25, - "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 815, - "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 20, - "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 21, - "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 197, - "apps/webapp/test/mollifierSyntheticTrace.test.ts": 303, - "apps/webapp/test/mollifierTripEvaluator.test.ts": 831, - "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 30311, - "apps/webapp/test/objectStore.test.ts": 5520, - "apps/webapp/test/orgBanner.test.ts": 18, - "apps/webapp/test/organizationDataStoresRegistry.test.ts": 5455, - "apps/webapp/test/otlpExporter.test.ts": 144, - "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 5831, - "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 317, - "apps/webapp/test/pauseEnvironment.server.test.ts": 8847, - "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 6886, - "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 16145, - "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 6231, - "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 36865, - "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 21, - "apps/webapp/test/prismaErrors.test.ts": 219, - "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 3592, - "apps/webapp/test/promptOverrideSource.test.ts": 17, - "apps/webapp/test/queryResultsTimeTicks.test.ts": 650, - "apps/webapp/test/queueListPagination.test.ts": 31, - "apps/webapp/test/rbacFallbackBranch.test.ts": 3600, - "apps/webapp/test/realtime/boundedTtlCache.test.ts": 22, - "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 41562, - "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 68, - "apps/webapp/test/realtime/envChangeRouter.test.ts": 1176, - "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 4496, - "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 342, - "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 607, - "apps/webapp/test/realtime/replayCursorStore.test.ts": 1529, - "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 584, - "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3540, - "apps/webapp/test/realtime/runReaderProjection.test.ts": 53, - "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 7242, - "apps/webapp/test/realtime/shadowCompare.test.ts": 26, - "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 5726, - "apps/webapp/test/redisRealtimeStreams.test.ts": 5745, - "apps/webapp/test/registryConfig.test.ts": 343, - "apps/webapp/test/reloadingRegistry.test.ts": 362, - "apps/webapp/test/removeTeamMember.test.ts": 9473, - "apps/webapp/test/replay-after-crash.test.ts": 2002, - "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 4349, - "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 5698, - "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 6634, - "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 5232, - "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 6025, - "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 550, - "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 5799, - "apps/webapp/test/runEngineHandlers.test.ts": 14088, - "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 348, - "apps/webapp/test/runOpsDbTopology.test.ts": 4214, - "apps/webapp/test/runOpsMintCutover.test.ts": 3808, - "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3407, - "apps/webapp/test/runOpsSplitMode.test.ts": 4202, - "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 473, - "apps/webapp/test/runOpsSplitReadGate.test.ts": 18, - "apps/webapp/test/runPresenterReadRoute.test.ts": 3909, - "apps/webapp/test/runsBackfiller.test.ts": 8095, - "apps/webapp/test/runsReplicationInstance.test.ts": 24640, - "apps/webapp/test/runsReplicationService.part1.test.ts": 25612, - "apps/webapp/test/runsReplicationService.part2.test.ts": 22323, - "apps/webapp/test/runsReplicationService.part3.test.ts": 12303, - "apps/webapp/test/runsReplicationService.part4.test.ts": 27040, - "apps/webapp/test/runsReplicationService.part5.test.ts": 9175, - "apps/webapp/test/runsReplicationService.part6.test.ts": 12682, - "apps/webapp/test/runsReplicationService.part7.test.ts": 69508, - "apps/webapp/test/runsReplicationService.part8.test.ts": 24479, - "apps/webapp/test/runsReplicationService.part9.test.ts": 12463, - "apps/webapp/test/runsRepository.part1.test.ts": 22987, - "apps/webapp/test/runsRepository.part2.test.ts": 23786, - "apps/webapp/test/runsRepository.part3.test.ts": 18531, - "apps/webapp/test/runsRepository.part4.test.ts": 24073, - "apps/webapp/test/runsRepository.readthrough.test.ts": 39432, - "apps/webapp/test/runsRepositoryCpres.test.ts": 8672, - "apps/webapp/test/runsRepositoryCursor.test.ts": 28582, - "apps/webapp/test/safeEnvironmentLog.test.ts": 17, - "apps/webapp/test/safeIntegrationLog.test.ts": 15, - "apps/webapp/test/safeRequestLogContext.test.ts": 26, - "apps/webapp/test/safeWebhookFetch.test.ts": 199, - "apps/webapp/test/safeWebhookUrl.test.ts": 30, - "apps/webapp/test/sameOriginNavigation.test.ts": 53, - "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 30, - "apps/webapp/test/sanitizeUrl.test.ts": 22, - "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 280, - "apps/webapp/test/sentryTenantContext.test.ts": 23, - "apps/webapp/test/sentryTraceContext.server.test.ts": 70, - "apps/webapp/test/services.controlPlane.readthrough.test.ts": 5401, - "apps/webapp/test/services/organizationAccessToken.test.ts": 241, - "apps/webapp/test/services/personalAccessToken.test.ts": 232, - "apps/webapp/test/sessionDuration.test.ts": 10233, - "apps/webapp/test/sessions.readthrough.test.ts": 5859, - "apps/webapp/test/sessionsReplicationService.test.ts": 17051, - "apps/webapp/test/shouldRevalidateRunsList.test.ts": 19, - "apps/webapp/test/slackOAuthResultLog.test.ts": 23, - "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 5373, - "apps/webapp/test/streamLoader.controlPlane.test.ts": 5162, - "apps/webapp/test/tenantContext.test.ts": 43, - "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 38, - "apps/webapp/test/tenantContextResolver.test.ts": 37, - "apps/webapp/test/timeGranularity.test.ts": 27, - "apps/webapp/test/timelineSpanEvents.test.ts": 25, - "apps/webapp/test/traceExport.test.ts": 29, - "apps/webapp/test/updateMetadata.test.ts": 19006, - "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 7309, - "apps/webapp/test/utils/timezones.test.ts": 31, - "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 5949, - "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 6678, - "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 5388, - "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 4587, - "apps/webapp/test/validateGitBranchName.test.ts": 23, - "apps/webapp/test/vercelUrls.test.ts": 18, - "apps/webapp/test/verifyDeploymentImage.test.ts": 922, - "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 7670, - "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 8556, - "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 5386, - "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 8112, - "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 4994, - "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 5687, - "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 35556, - "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 5062, - "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 5653, - "apps/webapp/test/webhookErrorAlerts.test.ts": 61, - "apps/webapp/test/workerGroupAccess.test.ts": 33, - "apps/webapp/test/workerQueueSplit.server.test.ts": 23, - "apps/webapp/test/workerQueueSplit.test.ts": 27, - "apps/webapp/test/workerRegions.test.ts": 487, + "apps/webapp/test/cancelRouteReplicaLag.guard.test.ts": 9736, + "apps/webapp/test/chartActivityTimeAxis.test.ts": 21, + "apps/webapp/test/chartXAxisTicks.test.ts": 7, + "apps/webapp/test/chartZoomRange.test.ts": 4, + "apps/webapp/test/chat-snapshot-integration.test.ts": 764, + "apps/webapp/test/checkPermissions.test.ts": 2, + "apps/webapp/test/checkSchedule.test.ts": 10509, + "apps/webapp/test/claimTtl.test.ts": 2, + "apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts": 608, + "apps/webapp/test/clickhouseFactory.test.ts": 7018, + "apps/webapp/test/components/DateTime.test.ts": 18, + "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 7, + "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 84, + "apps/webapp/test/components/runs/v3/RunTag.test.ts": 4, + "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 2, + "apps/webapp/test/components/webhookDeliveries/buildDeliveryTimelineItems.test.ts": 4, + "apps/webapp/test/computeBucket.test.ts": 97, + "apps/webapp/test/computeMigration.test.ts": 2, + "apps/webapp/test/concurrencySystemPercentOverride.test.ts": 10891, + "apps/webapp/test/concurrentFlushScheduler.test.ts": 369, + "apps/webapp/test/contextlessPatRoutes.test.ts": 29, + "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 4413, + "apps/webapp/test/createEnvironmentApiKey.test.ts": 10169, + "apps/webapp/test/crossSeamGuard.proof.test.ts": 9872, + "apps/webapp/test/dashboardAgentAlertAdminPreview.test.ts": 2, + "apps/webapp/test/dashboardAgentBodyCap.test.ts": 83, + "apps/webapp/test/dashboardAgentChatRetention.test.ts": 1550, + "apps/webapp/test/dashboardAgentClientMetadata.test.ts": 15, + "apps/webapp/test/dashboardAgentCreateChatOrdering.test.ts": 11, + "apps/webapp/test/dashboardAgentDurableResume.test.ts": 5441, + "apps/webapp/test/dashboardAgentEvalPolicyAuth.test.ts": 75, + "apps/webapp/test/dashboardAgentForeignChat.test.ts": 11, + "apps/webapp/test/dashboardAgentHeadStart.test.ts": 3, + "apps/webapp/test/dashboardAgentImageCsp.test.ts": 2, + "apps/webapp/test/dashboardAgentInProxyMintFailure.test.ts": 6, + "apps/webapp/test/dashboardAgentInvestigationSettlementCard.test.ts": 5, + "apps/webapp/test/dashboardAgentInvestigationWinner.test.ts": 2, + "apps/webapp/test/dashboardAgentLastReadBackfill.test.ts": 11790, + "apps/webapp/test/dashboardAgentLegacyMessagesColumn.test.ts": 396, + "apps/webapp/test/dashboardAgentMessageCards.test.ts": 12, + "apps/webapp/test/dashboardAgentMessageSurrogate.test.ts": 2556, + "apps/webapp/test/dashboardAgentQueriesTenantIsolation.test.ts": 5634, + "apps/webapp/test/dashboardAgentQuota.test.ts": 17173, + "apps/webapp/test/dashboardAgentRoutes.test.ts": 22, + "apps/webapp/test/dashboardAgentSurrogatePersist.test.ts": 6426, + "apps/webapp/test/dashboardAgentTenantIsolation.test.ts": 11316, + "apps/webapp/test/dashboardAgentToolScopes.test.ts": 1, + "apps/webapp/test/dashboardAgentTranscriptStore.test.ts": 22820, + "apps/webapp/test/dashboardAgentUnreadWorkScope.test.ts": 1730, + "apps/webapp/test/dashboardAgentWakeActivity.test.ts": 2441, + "apps/webapp/test/dashboardAgentWatchAlertEnvironments.test.ts": 3514, + "apps/webapp/test/dashboardAgentWatchAlertFanout.test.ts": 16, + "apps/webapp/test/dashboardAgentWatchAlertGate.test.ts": 2, + "apps/webapp/test/dashboardAgentWatchAlertOwnerScope.test.ts": 1725, + "apps/webapp/test/dashboardAgentWatchBatchFairness.test.ts": 3909, + "apps/webapp/test/dashboardAgentWatchBatchRecording.test.ts": 8074, + "apps/webapp/test/dashboardAgentWatchCardAtomicity.test.ts": 7569, + "apps/webapp/test/dashboardAgentWatchCardRequestId.test.ts": 880, + "apps/webapp/test/dashboardAgentWatchChecks.test.ts": 7, + "apps/webapp/test/dashboardAgentWatchCreationReads.test.ts": 2, + "apps/webapp/test/dashboardAgentWatchErrorFingerprint.test.ts": 4824, + "apps/webapp/test/dashboardAgentWatchInvestigate.test.ts": 18, + "apps/webapp/test/dashboardAgentWatchLimitStatus.test.ts": 1830, + "apps/webapp/test/dashboardAgentWatchLimits.test.ts": 14463, + "apps/webapp/test/dashboardAgentWatchQueueAge.test.ts": 2, + "apps/webapp/test/dashboardAgentWatchQueueName.test.ts": 14784, + "apps/webapp/test/dashboardAgentWatchSweepAlertOnce.test.ts": 1237, + "apps/webapp/test/dashboardAgentWatchSweepBoundary.test.ts": 8660, + "apps/webapp/test/dashboardAgentWatchTenancy.test.ts": 5475, + "apps/webapp/test/dashboardAgentWatchToken.test.ts": 16, + "apps/webapp/test/dashboardAgentWatchWording.test.ts": 7, + "apps/webapp/test/dashboardAgentWatches.test.ts": 109690, + "apps/webapp/test/deleteTaskSchedule.test.ts": 3072, + "apps/webapp/test/deliveryIdBounds.test.ts": 18, + "apps/webapp/test/dependentAttemptScope.test.ts": 2, + "apps/webapp/test/deploymentApiPaths.test.ts": 2, + "apps/webapp/test/detectQueryTables.test.ts": 97, + "apps/webapp/test/detectbadJsonStrings.test.ts": 81, + "apps/webapp/test/devBranchServices.test.ts": 3555, + "apps/webapp/test/devPresenceRecency.test.ts": 168, + "apps/webapp/test/directorySyncEffects.server.test.ts": 7, + "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2465, + "apps/webapp/test/duplicateTaskIds.test.ts": 3, + "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1532, + "apps/webapp/test/emailPattern.test.ts": 4, + "apps/webapp/test/engine/batchPayloads.test.ts": 5017, + "apps/webapp/test/engine/dequeueWorkerVersionFreshness.test.ts": 3524, + "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 2375, + "apps/webapp/test/engine/streamBatchItems.test.ts": 50389, + "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 7090, + "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 2665, + "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 2320, + "apps/webapp/test/engine/triggerTask.debounce.test.ts": 5728, + "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 4896, + "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 4620, + "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 4192, + "apps/webapp/test/engine/triggerTask.residency.test.ts": 3482, + "apps/webapp/test/engine/triggerTask.test.ts": 5846, + "apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts": 8985, + "apps/webapp/test/env.server.test.ts": 500, + "apps/webapp/test/envConcurrencyLimitPause.server.test.ts": 7727, + "apps/webapp/test/envJwtActorClaim.test.ts": 20, + "apps/webapp/test/envParamRoute.ownership.test.ts": 5, + "apps/webapp/test/environmentSort.test.ts": 8, + "apps/webapp/test/environmentVariableApiAccess.test.ts": 7, + "apps/webapp/test/environmentVariableDeduplication.test.ts": 3, + "apps/webapp/test/environmentVariableRules.test.ts": 2, + "apps/webapp/test/environmentVariablesEnvironments.test.ts": 5637, + "apps/webapp/test/environmentVariablesReplicaRouting.test.ts": 894, + "apps/webapp/test/environmentVariablesRepository.test.ts": 4139, + "apps/webapp/test/errorFingerprinting.test.ts": 5, + "apps/webapp/test/errorGroupWebhook.test.ts": 10, + "apps/webapp/test/featureFlags.test.ts": 7936, + "apps/webapp/test/findEnvironmentByApiKey.test.ts": 10783, + "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 15826, + "apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts": 6053, + "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 3627, + "apps/webapp/test/getDeploymentImageRef.test.ts": 6, + "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 466, + "apps/webapp/test/googleEmailVerification.test.ts": 2, + "apps/webapp/test/httpErrors.test.ts": 2, + "apps/webapp/test/idempotencyDedupResidency.test.ts": 20592, + "apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts": 2, + "apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts": 32211, + "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 17731, + "apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts": 14016, + "apps/webapp/test/impersonationConsent.test.ts": 8172, + "apps/webapp/test/internalApiOrigin.test.ts": 2, + "apps/webapp/test/inviteRoleLadder.test.ts": 2, + "apps/webapp/test/logger.server.onError.test.ts": 37, + "apps/webapp/test/logsSearchProjector.test.ts": 7, + "apps/webapp/test/logsSearchProjectorRedisStore.test.ts": 36, + "apps/webapp/test/logsSearchProjectorStateStore.test.ts": 729, + "apps/webapp/test/member.server.test.ts": 49667, + "apps/webapp/test/memberDevEnvironments.server.test.ts": 7442, + "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 5, + "apps/webapp/test/metadataRouteReplicaLag.guard.test.ts": 13, + "apps/webapp/test/mfaRateLimiter.test.ts": 331, + "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 494, + "apps/webapp/test/mollifierClaimResolution.test.ts": 4, + "apps/webapp/test/mollifierDecisionLabels.test.ts": 2, + "apps/webapp/test/mollifierDrainerHandler.test.ts": 19, + "apps/webapp/test/mollifierDrainerWorker.test.ts": 4, + "apps/webapp/test/mollifierDrainingGauge.test.ts": 448, + "apps/webapp/test/mollifierGate.test.ts": 11, + "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 16, + "apps/webapp/test/mollifierMollify.test.ts": 6, + "apps/webapp/test/mollifierMutateWithFallback.test.ts": 4, + "apps/webapp/test/mollifierReadFallback.test.ts": 11, + "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 3, + "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 8, + "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 6, + "apps/webapp/test/mollifierStaleSweep.test.ts": 1030, + "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 3, + "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 2, + "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 63, + "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 2, + "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 1, + "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 5, + "apps/webapp/test/mollifierSyntheticTrace.test.ts": 4, + "apps/webapp/test/mollifierTripEvaluator.test.ts": 54, + "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 28379, + "apps/webapp/test/objectStore.test.ts": 11175, + "apps/webapp/test/orgBanner.test.ts": 2, + "apps/webapp/test/orgMember.server.test.ts": 3927, + "apps/webapp/test/organizationDataStoresRegistry.test.ts": 31981, + "apps/webapp/test/otlpExporter.test.ts": 6, + "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 146, + "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 937, + "apps/webapp/test/pauseEnvironment.server.test.ts": 3306, + "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 12157, + "apps/webapp/test/platformNotifications.test.ts": 8, + "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 2118, + "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 100, + "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 38297, + "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 3, + "apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts": 14584, + "apps/webapp/test/prismaErrors.test.ts": 1, + "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 4980, + "apps/webapp/test/projectEnvironmentCredentialRoute.test.ts": 5, + "apps/webapp/test/projectEnvironmentsBranchScope.test.ts": 46, + "apps/webapp/test/projectSettingsToastRedirect.test.ts": 12, + "apps/webapp/test/promptOverrideSource.test.ts": 2, + "apps/webapp/test/publicAccessTokenResponse.test.ts": 48, + "apps/webapp/test/publicTokensRoute.test.ts": 1556, + "apps/webapp/test/publishClaimResult.test.ts": 1, + "apps/webapp/test/queryResultsTimeTicks.test.ts": 2, + "apps/webapp/test/queryRouteReadOnly.test.ts": 63, + "apps/webapp/test/queryScope.test.ts": 3, + "apps/webapp/test/queueDepthSeries.test.ts": 6, + "apps/webapp/test/queueListPagination.test.ts": 2, + "apps/webapp/test/queueMetricsMapping.test.ts": 6, + "apps/webapp/test/queueRetrieveJwt.test.ts": 40, + "apps/webapp/test/queueSparklineGrid.test.ts": 6, + "apps/webapp/test/rbacFallbackBranch.test.ts": 40147, + "apps/webapp/test/rbacFallbackSessionFloor.test.ts": 13998, + "apps/webapp/test/reacquireClearedGlobalWinner.test.ts": 3, + "apps/webapp/test/readBodyWithCap.test.ts": 7, + "apps/webapp/test/readRunForEvent.replicaLag.test.ts": 2497, + "apps/webapp/test/realtime/boundedTtlCache.test.ts": 4, + "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 55110, + "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 8, + "apps/webapp/test/realtime/envChangeRouter.test.ts": 959, + "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 3659, + "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 7, + "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 306, + "apps/webapp/test/realtime/replayCursorStore.test.ts": 1443, + "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 376, + "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3003, + "apps/webapp/test/realtime/runReaderProjection.test.ts": 3, + "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 31887, + "apps/webapp/test/realtime/shadowCompare.test.ts": 7, + "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 14199, + "apps/webapp/test/realtimeClient.test.ts": 1, + "apps/webapp/test/realtimeServices.replicaLag.test.ts": 9845, + "apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts": 7384, + "apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts": 18951, + "apps/webapp/test/realtimeStreamsVersion.test.ts": 3, + "apps/webapp/test/redisRealtimeStreams.test.ts": 5306, + "apps/webapp/test/registryConfig.test.ts": 298, + "apps/webapp/test/reloadingRegistry.test.ts": 2, + "apps/webapp/test/removeTeamMember.test.ts": 6776, + "apps/webapp/test/replay-after-crash.test.ts": 989, + "apps/webapp/test/replayRouteReplicaLag.guard.test.ts": 13268, + "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 2602, + "apps/webapp/test/reportCurationTrust.test.ts": 6, + "apps/webapp/test/reportHealth.test.ts": 16, + "apps/webapp/test/reportHealthData.test.ts": 10, + "apps/webapp/test/reportMetricDelta.test.ts": 20, + "apps/webapp/test/reportPresenter.test.ts": 25, + "apps/webapp/test/reportRenderParity.test.ts": 75, + "apps/webapp/test/reportTrust.test.ts": 2, + "apps/webapp/test/reportsApiRoute.test.ts": 12, + "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 14311, + "apps/webapp/test/resolveBatchForRealtime.test.ts": 1, + "apps/webapp/test/resolveOrgIdFromSlugForUser.test.ts": 2355, + "apps/webapp/test/resolveProjectScopedEnvironments.test.ts": 4, + "apps/webapp/test/resolveTriggerUri.test.ts": 6, + "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 16837, + "apps/webapp/test/routeCspImgSrc.test.ts": 34, + "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 10139, + "apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts": 22148, + "apps/webapp/test/runCommitAuthorization.test.ts": 8, + "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 13211, + "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 2, + "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 4283, + "apps/webapp/test/runEngineHandlers.test.ts": 18783, + "apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts": 27371, + "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 10, + "apps/webapp/test/runOpsDbTopology.test.ts": 21098, + "apps/webapp/test/runOpsMintCutover.test.ts": 3734, + "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3972, + "apps/webapp/test/runOpsSplitMode.test.ts": 13489, + "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 2, + "apps/webapp/test/runOpsSplitReadGate.test.ts": 4, + "apps/webapp/test/runPresenterReadRoute.test.ts": 4891, + "apps/webapp/test/runPresenters.replicaLag.test.ts": 25923, + "apps/webapp/test/runTimestamps.test.ts": 2, + "apps/webapp/test/runsBackfiller.test.ts": 13335, + "apps/webapp/test/runsReplicationBenchmark.test.ts": 1, + "apps/webapp/test/runsReplicationInstance.test.ts": 22824, + "apps/webapp/test/runsReplicationJsonRecoveryBenchmark.test.ts": 1, + "apps/webapp/test/runsReplicationService.part1.test.ts": 32108, + "apps/webapp/test/runsReplicationService.part10.test.ts": 21422, + "apps/webapp/test/runsReplicationService.part2.test.ts": 35180, + "apps/webapp/test/runsReplicationService.part3.test.ts": 20895, + "apps/webapp/test/runsReplicationService.part4.test.ts": 34760, + "apps/webapp/test/runsReplicationService.part5.test.ts": 9752, + "apps/webapp/test/runsReplicationService.part6.test.ts": 20139, + "apps/webapp/test/runsReplicationService.part7.test.ts": 70185, + "apps/webapp/test/runsReplicationService.part8.test.ts": 29146, + "apps/webapp/test/runsReplicationService.part9.test.ts": 8107, + "apps/webapp/test/runsRepository.part1.test.ts": 31337, + "apps/webapp/test/runsRepository.part2.test.ts": 39180, + "apps/webapp/test/runsRepository.part3.test.ts": 30925, + "apps/webapp/test/runsRepository.part4.test.ts": 36000, + "apps/webapp/test/runsRepository.readthrough.test.ts": 48205, + "apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts": 5349, + "apps/webapp/test/runsRepositoryCpres.test.ts": 14258, + "apps/webapp/test/runsRepositoryCursor.test.ts": 49050, + "apps/webapp/test/safeEnvironmentLog.test.ts": 1, + "apps/webapp/test/safeIntegrationLog.test.ts": 2, + "apps/webapp/test/safeRequestLogContext.test.ts": 4, + "apps/webapp/test/safeWebhookFetch.test.ts": 5, + "apps/webapp/test/safeWebhookUrl.test.ts": 6, + "apps/webapp/test/sameOriginNavigation.test.ts": 2, + "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 9, + "apps/webapp/test/sanitizeSessionInput.server.test.ts": 3, + "apps/webapp/test/sanitizeUrl.test.ts": 2, + "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 4, + "apps/webapp/test/scheduleTimings.test.ts": 1506, + "apps/webapp/test/scheduleWindow.test.ts": 31, + "apps/webapp/test/schedulesPutEnvScoping.test.ts": 15522, + "apps/webapp/test/selectBestEnvironment.test.ts": 1, + "apps/webapp/test/sentryRequestIsolation.test.ts": 65, + "apps/webapp/test/sentryTenantContext.test.ts": 4, + "apps/webapp/test/sentryTraceContext.server.test.ts": 9, + "apps/webapp/test/services.controlPlane.readthrough.test.ts": 21682, + "apps/webapp/test/services/organizationAccessToken.test.ts": 6, + "apps/webapp/test/services/personalAccessToken.test.ts": 7, + "apps/webapp/test/sessionDuration.test.ts": 16345, + "apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts": 13790, + "apps/webapp/test/sessions.readthrough.test.ts": 9480, + "apps/webapp/test/sessionsReplicationService.test.ts": 22809, + "apps/webapp/test/setActiveOnTaskSchedule.test.ts": 7851, + "apps/webapp/test/shouldRevalidateRunsList.test.ts": 2, + "apps/webapp/test/slackErrorAlerts.test.ts": 1, + "apps/webapp/test/slackOAuthResultLog.test.ts": 1, + "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 11747, + "apps/webapp/test/spanTraceRoutes.replicaLag.test.ts": 15558, + "apps/webapp/test/streamBatchItemsAuthorization.test.ts": 4, + "apps/webapp/test/streamLoader.controlPlane.test.ts": 14703, + "apps/webapp/test/syncDeclarativeSchedules.test.ts": 3536, + "apps/webapp/test/syncDeclarativeWebhooks.test.ts": 5631, + "apps/webapp/test/taskCodeSnippets.test.ts": 5, + "apps/webapp/test/tenantContext.test.ts": 27, + "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 3, + "apps/webapp/test/tenantContextResolver.test.ts": 16, + "apps/webapp/test/themePreference.test.ts": 4, + "apps/webapp/test/timeGranularity.test.ts": 3, + "apps/webapp/test/timelineSpanEvents.test.ts": 3, + "apps/webapp/test/traceExport.test.ts": 7, + "apps/webapp/test/uatEnvironmentClaim.test.ts": 22, + "apps/webapp/test/updateMetadata.test.ts": 14977, + "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 14921, + "apps/webapp/test/useTableSort.test.ts": 9, + "apps/webapp/test/userActorEnvironmentScopeRouteBuilder.test.ts": 42, + "apps/webapp/test/userActorPatOnlyBoundary.test.ts": 4565, + "apps/webapp/test/userActorProjectWideScope.test.ts": 9578, + "apps/webapp/test/userActorSourcePat.test.ts": 85, + "apps/webapp/test/userActorTokenClaimsAndScopes.test.ts": 3360, + "apps/webapp/test/utils/timezones.test.ts": 27, + "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 15923, + "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 16427, + "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 19703, + "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 17785, + "apps/webapp/test/validateGitBranchName.test.ts": 5, + "apps/webapp/test/vercelUrls.test.ts": 4, + "apps/webapp/test/verifyDeploymentImage.test.ts": 621, + "apps/webapp/test/viewAsUser.test.ts": 5, + "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 15766, + "apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts": 7980, + "apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts": 17659, + "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 10384, + "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 16964, + "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 17630, + "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 15593, + "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 17406, + "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 22650, + "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 9618, + "apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts": 29510, + "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 12113, + "apps/webapp/test/webhookErrorAlerts.test.ts": 4, + "apps/webapp/test/workerGroupAccess.test.ts": 2, + "apps/webapp/test/workerIdUnwrap.test.ts": 47, + "apps/webapp/test/workerQueueSplit.server.test.ts": 3, + "apps/webapp/test/workerQueueSplit.test.ts": 4, + "apps/webapp/test/workerRegions.test.ts": 3, + "apps/webapp/test/workloadTokenAuthorization.test.ts": 1, + "apps/webapp/test/workloadTokenGate.integration.test.ts": 2397, + "apps/webapp/test/writableEnvironments.test.ts": 1, "internal-packages/cache/src/stores/lruMemory.test.ts": 65, "internal-packages/clickhouse/src/client/client.test.ts": 7547, "internal-packages/clickhouse/src/taskRuns.test.ts": 6768, From f62351a7808520f5e6479a7d8551619922d64c9d Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 16:08:58 +0100 Subject: [PATCH 02/15] perf(ci): calibrate webapp shards with CI timings --- .github/workflows/unit-tests-webapp.yml | 3 +- test-timings.json | 200 ++++++++++++------------ 2 files changed, 102 insertions(+), 101 deletions(-) diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index 4db1302bdaa..0a3fa28b68c 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -23,7 +23,8 @@ jobs: # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard fail-fast: false matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + shardIndex: + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] shardTotal: [24] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/test-timings.json b/test-timings.json index 0dca9f10562..13406562eee 100644 --- a/test-timings.json +++ b/test-timings.json @@ -65,10 +65,10 @@ "apps/webapp/app/routes/storybook.agent-ui/fixtures.test.ts": 6, "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 3, "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 2, - "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 796, - "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 2972, - "apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts": 2896, - "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 2743, + "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 50313, + "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 132163, + "apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts": 91190, + "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 132085, "apps/webapp/app/utils/apiKeys.test.ts": 6, "apps/webapp/app/utils/boundedRequestBody.server.test.ts": 9, "apps/webapp/app/utils/cspImageOrigins.test.ts": 3, @@ -93,62 +93,62 @@ "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 2, "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 5, "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 13, - "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 12533, + "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 8268, "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 1, "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 10, "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 5, - "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 15874, - "apps/webapp/app/v3/runStore.server.test.ts": 13921, + "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 8273, + "apps/webapp/app/v3/runStore.server.test.ts": 8656, "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 22710, "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 3, "apps/webapp/app/v3/utils/priority.test.ts": 2, "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 5039, "apps/webapp/test/GCRARateLimiter.test.ts": 4477, - "apps/webapp/test/SpanPresenter.readthrough.test.ts": 11106, + "apps/webapp/test/SpanPresenter.readthrough.test.ts": 9160, "apps/webapp/test/activitySeries.server.test.ts": 6, "apps/webapp/test/additionalApiKeyIssuance.test.ts": 3, "apps/webapp/test/aiTitleRateLimiter.test.ts": 55, - "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 8260, - "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 10265, + "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 94856, + "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 174422, "apps/webapp/test/apiAuthActorClaim.test.ts": 4, "apps/webapp/test/apiAuthScope.test.ts": 7, "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 16921, - "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 15142, - "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 18108, + "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 7297, + "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 8909, "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 1130, "apps/webapp/test/apiBuilderAuthorization.test.ts": 1, - "apps/webapp/test/apiKeysPresenter.test.ts": 4705, + "apps/webapp/test/apiKeysPresenter.test.ts": 10287, "apps/webapp/test/apiRateLimitJwtActor.test.ts": 4, "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 946, - "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 9845, - "apps/webapp/test/apiRunListPresenter.test.ts": 74771, - "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 18402, + "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 8834, + "apps/webapp/test/apiRunListPresenter.test.ts": 141836, + "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 9048, "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 1713, - "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 21866, + "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 11841, "apps/webapp/test/authFeatureControls.test.ts": 3, - "apps/webapp/test/authorizationCodeConsent.test.ts": 4459, + "apps/webapp/test/authorizationCodeConsent.test.ts": 10286, "apps/webapp/test/authorizationRateLimitMiddleware.test.ts": 1, "apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts": 107, - "apps/webapp/test/batchListPresenter.readroute.test.ts": 26765, - "apps/webapp/test/batchPresenter.test.ts": 20446, + "apps/webapp/test/batchListPresenter.readroute.test.ts": 12433, + "apps/webapp/test/batchPresenter.test.ts": 15117, "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 6, - "apps/webapp/test/batchRunAccess.test.ts": 4049, + "apps/webapp/test/batchRunAccess.test.ts": 9831, "apps/webapp/test/batchServices.replicaLag.test.ts": 20440, "apps/webapp/test/batchStreamGrants.test.ts": 199, - "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 8680, + "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 8523, "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 5, - "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 14259, + "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 8063, "apps/webapp/test/billingAlertsDefaults.test.ts": 2, "apps/webapp/test/billingAlertsFormat.test.ts": 2, "apps/webapp/test/billingLimit.schemas.test.ts": 7, - "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 13473, + "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 16050, "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 6177, "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 2, "apps/webapp/test/billingLimitConvergeResolve.test.ts": 15, "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 5, "apps/webapp/test/billingLimitHit.test.ts": 3, "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 2, - "apps/webapp/test/billingLimitQueuedRuns.test.ts": 29233, + "apps/webapp/test/billingLimitQueuedRuns.test.ts": 18328, "apps/webapp/test/billingLimitReconcileTick.test.ts": 7, "apps/webapp/test/billingLimitReconciliation.test.ts": 13124, "apps/webapp/test/billingLimitResolve.test.ts": 2, @@ -157,7 +157,7 @@ "apps/webapp/test/branchableEnvironment.test.ts": 3, "apps/webapp/test/bufferedTriggerPayload.test.ts": 4, "apps/webapp/test/bulkActionV2.replicaLag.test.ts": 8967, - "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 15978, + "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 8014, "apps/webapp/test/calculateNextSchedule.test.ts": 208, "apps/webapp/test/cancelRouteReplicaLag.guard.test.ts": 9736, "apps/webapp/test/chartActivityTimeAxis.test.ts": 21, @@ -165,7 +165,7 @@ "apps/webapp/test/chartZoomRange.test.ts": 4, "apps/webapp/test/chat-snapshot-integration.test.ts": 764, "apps/webapp/test/checkPermissions.test.ts": 2, - "apps/webapp/test/checkSchedule.test.ts": 10509, + "apps/webapp/test/checkSchedule.test.ts": 10866, "apps/webapp/test/claimTtl.test.ts": 2, "apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts": 608, "apps/webapp/test/clickhouseFactory.test.ts": 7018, @@ -180,8 +180,8 @@ "apps/webapp/test/concurrencySystemPercentOverride.test.ts": 10891, "apps/webapp/test/concurrentFlushScheduler.test.ts": 369, "apps/webapp/test/contextlessPatRoutes.test.ts": 29, - "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 4413, - "apps/webapp/test/createEnvironmentApiKey.test.ts": 10169, + "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 10196, + "apps/webapp/test/createEnvironmentApiKey.test.ts": 11666, "apps/webapp/test/crossSeamGuard.proof.test.ts": 9872, "apps/webapp/test/dashboardAgentAlertAdminPreview.test.ts": 2, "apps/webapp/test/dashboardAgentBodyCap.test.ts": 83, @@ -206,7 +206,7 @@ "apps/webapp/test/dashboardAgentSurrogatePersist.test.ts": 6426, "apps/webapp/test/dashboardAgentTenantIsolation.test.ts": 11316, "apps/webapp/test/dashboardAgentToolScopes.test.ts": 1, - "apps/webapp/test/dashboardAgentTranscriptStore.test.ts": 22820, + "apps/webapp/test/dashboardAgentTranscriptStore.test.ts": 11851, "apps/webapp/test/dashboardAgentUnreadWorkScope.test.ts": 1730, "apps/webapp/test/dashboardAgentWakeActivity.test.ts": 2441, "apps/webapp/test/dashboardAgentWatchAlertEnvironments.test.ts": 3514, @@ -230,8 +230,8 @@ "apps/webapp/test/dashboardAgentWatchTenancy.test.ts": 5475, "apps/webapp/test/dashboardAgentWatchToken.test.ts": 16, "apps/webapp/test/dashboardAgentWatchWording.test.ts": 7, - "apps/webapp/test/dashboardAgentWatches.test.ts": 109690, - "apps/webapp/test/deleteTaskSchedule.test.ts": 3072, + "apps/webapp/test/dashboardAgentWatches.test.ts": 123752, + "apps/webapp/test/deleteTaskSchedule.test.ts": 16973, "apps/webapp/test/deliveryIdBounds.test.ts": 18, "apps/webapp/test/dependentAttemptScope.test.ts": 2, "apps/webapp/test/deploymentApiPaths.test.ts": 2, @@ -245,21 +245,21 @@ "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1532, "apps/webapp/test/emailPattern.test.ts": 4, "apps/webapp/test/engine/batchPayloads.test.ts": 5017, - "apps/webapp/test/engine/dequeueWorkerVersionFreshness.test.ts": 3524, - "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 2375, - "apps/webapp/test/engine/streamBatchItems.test.ts": 50389, + "apps/webapp/test/engine/dequeueWorkerVersionFreshness.test.ts": 10262, + "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 11035, + "apps/webapp/test/engine/streamBatchItems.test.ts": 22941, "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 7090, - "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 2665, - "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 2320, - "apps/webapp/test/engine/triggerTask.debounce.test.ts": 5728, - "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 4896, - "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 4620, - "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 4192, - "apps/webapp/test/engine/triggerTask.residency.test.ts": 3482, - "apps/webapp/test/engine/triggerTask.test.ts": 5846, + "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 132100, + "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 90077, + "apps/webapp/test/engine/triggerTask.debounce.test.ts": 174383, + "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133924, + "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 172695, + "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 172753, + "apps/webapp/test/engine/triggerTask.residency.test.ts": 172776, + "apps/webapp/test/engine/triggerTask.test.ts": 132169, "apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts": 8985, "apps/webapp/test/env.server.test.ts": 500, - "apps/webapp/test/envConcurrencyLimitPause.server.test.ts": 7727, + "apps/webapp/test/envConcurrencyLimitPause.server.test.ts": 214032, "apps/webapp/test/envJwtActorClaim.test.ts": 20, "apps/webapp/test/envParamRoute.ownership.test.ts": 5, "apps/webapp/test/environmentSort.test.ts": 8, @@ -267,7 +267,7 @@ "apps/webapp/test/environmentVariableDeduplication.test.ts": 3, "apps/webapp/test/environmentVariableRules.test.ts": 2, "apps/webapp/test/environmentVariablesEnvironments.test.ts": 5637, - "apps/webapp/test/environmentVariablesReplicaRouting.test.ts": 894, + "apps/webapp/test/environmentVariablesReplicaRouting.test.ts": 9602, "apps/webapp/test/environmentVariablesRepository.test.ts": 4139, "apps/webapp/test/errorFingerprinting.test.ts": 5, "apps/webapp/test/errorGroupWebhook.test.ts": 10, @@ -275,24 +275,24 @@ "apps/webapp/test/findEnvironmentByApiKey.test.ts": 10783, "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 15826, "apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts": 6053, - "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 3627, + "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 9970, "apps/webapp/test/getDeploymentImageRef.test.ts": 6, "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 466, "apps/webapp/test/googleEmailVerification.test.ts": 2, "apps/webapp/test/httpErrors.test.ts": 2, - "apps/webapp/test/idempotencyDedupResidency.test.ts": 20592, + "apps/webapp/test/idempotencyDedupResidency.test.ts": 7600, "apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts": 2, - "apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts": 32211, + "apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts": 14676, "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 17731, "apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts": 14016, - "apps/webapp/test/impersonationConsent.test.ts": 8172, + "apps/webapp/test/impersonationConsent.test.ts": 10012, "apps/webapp/test/internalApiOrigin.test.ts": 2, "apps/webapp/test/inviteRoleLadder.test.ts": 2, "apps/webapp/test/logger.server.onError.test.ts": 37, "apps/webapp/test/logsSearchProjector.test.ts": 7, "apps/webapp/test/logsSearchProjectorRedisStore.test.ts": 36, "apps/webapp/test/logsSearchProjectorStateStore.test.ts": 729, - "apps/webapp/test/member.server.test.ts": 49667, + "apps/webapp/test/member.server.test.ts": 9006, "apps/webapp/test/memberDevEnvironments.server.test.ts": 7442, "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 5, "apps/webapp/test/metadataRouteReplicaLag.guard.test.ts": 13, @@ -320,7 +320,7 @@ "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 5, "apps/webapp/test/mollifierSyntheticTrace.test.ts": 4, "apps/webapp/test/mollifierTripEvaluator.test.ts": 54, - "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 28379, + "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 25633, "apps/webapp/test/objectStore.test.ts": 11175, "apps/webapp/test/orgBanner.test.ts": 2, "apps/webapp/test/orgMember.server.test.ts": 3927, @@ -328,12 +328,12 @@ "apps/webapp/test/otlpExporter.test.ts": 6, "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 146, "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 937, - "apps/webapp/test/pauseEnvironment.server.test.ts": 3306, - "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 12157, + "apps/webapp/test/pauseEnvironment.server.test.ts": 10560, + "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 7527, "apps/webapp/test/platformNotifications.test.ts": 8, - "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 2118, + "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 10293, "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 100, - "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 38297, + "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 37569, "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 3, "apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts": 14584, "apps/webapp/test/prismaErrors.test.ts": 1, @@ -353,13 +353,13 @@ "apps/webapp/test/queueMetricsMapping.test.ts": 6, "apps/webapp/test/queueRetrieveJwt.test.ts": 40, "apps/webapp/test/queueSparklineGrid.test.ts": 6, - "apps/webapp/test/rbacFallbackBranch.test.ts": 40147, + "apps/webapp/test/rbacFallbackBranch.test.ts": 8470, "apps/webapp/test/rbacFallbackSessionFloor.test.ts": 13998, "apps/webapp/test/reacquireClearedGlobalWinner.test.ts": 3, "apps/webapp/test/readBodyWithCap.test.ts": 7, - "apps/webapp/test/readRunForEvent.replicaLag.test.ts": 2497, + "apps/webapp/test/readRunForEvent.replicaLag.test.ts": 9913, "apps/webapp/test/realtime/boundedTtlCache.test.ts": 4, - "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 55110, + "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 51762, "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 8, "apps/webapp/test/realtime/envChangeRouter.test.ts": 959, "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 3659, @@ -369,9 +369,9 @@ "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 376, "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3003, "apps/webapp/test/realtime/runReaderProjection.test.ts": 3, - "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 31887, + "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 10077, "apps/webapp/test/realtime/shadowCompare.test.ts": 7, - "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 14199, + "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 7894, "apps/webapp/test/realtimeClient.test.ts": 1, "apps/webapp/test/realtimeServices.replicaLag.test.ts": 9845, "apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts": 7384, @@ -380,7 +380,7 @@ "apps/webapp/test/redisRealtimeStreams.test.ts": 5306, "apps/webapp/test/registryConfig.test.ts": 298, "apps/webapp/test/reloadingRegistry.test.ts": 2, - "apps/webapp/test/removeTeamMember.test.ts": 6776, + "apps/webapp/test/removeTeamMember.test.ts": 11245, "apps/webapp/test/replay-after-crash.test.ts": 989, "apps/webapp/test/replayRouteReplicaLag.guard.test.ts": 13268, "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 2602, @@ -397,16 +397,16 @@ "apps/webapp/test/resolveOrgIdFromSlugForUser.test.ts": 2355, "apps/webapp/test/resolveProjectScopedEnvironments.test.ts": 4, "apps/webapp/test/resolveTriggerUri.test.ts": 6, - "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 16837, + "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 7575, "apps/webapp/test/routeCspImgSrc.test.ts": 34, - "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 10139, + "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 7302, "apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts": 22148, "apps/webapp/test/runCommitAuthorization.test.ts": 8, "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 13211, "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 2, "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 4283, - "apps/webapp/test/runEngineHandlers.test.ts": 18783, - "apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts": 27371, + "apps/webapp/test/runEngineHandlers.test.ts": 16844, + "apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts": 8775, "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 10, "apps/webapp/test/runOpsDbTopology.test.ts": 21098, "apps/webapp/test/runOpsMintCutover.test.ts": 3734, @@ -417,28 +417,28 @@ "apps/webapp/test/runPresenterReadRoute.test.ts": 4891, "apps/webapp/test/runPresenters.replicaLag.test.ts": 25923, "apps/webapp/test/runTimestamps.test.ts": 2, - "apps/webapp/test/runsBackfiller.test.ts": 13335, + "apps/webapp/test/runsBackfiller.test.ts": 10511, "apps/webapp/test/runsReplicationBenchmark.test.ts": 1, - "apps/webapp/test/runsReplicationInstance.test.ts": 22824, + "apps/webapp/test/runsReplicationInstance.test.ts": 20278, "apps/webapp/test/runsReplicationJsonRecoveryBenchmark.test.ts": 1, - "apps/webapp/test/runsReplicationService.part1.test.ts": 32108, - "apps/webapp/test/runsReplicationService.part10.test.ts": 21422, - "apps/webapp/test/runsReplicationService.part2.test.ts": 35180, - "apps/webapp/test/runsReplicationService.part3.test.ts": 20895, - "apps/webapp/test/runsReplicationService.part4.test.ts": 34760, - "apps/webapp/test/runsReplicationService.part5.test.ts": 9752, - "apps/webapp/test/runsReplicationService.part6.test.ts": 20139, - "apps/webapp/test/runsReplicationService.part7.test.ts": 70185, - "apps/webapp/test/runsReplicationService.part8.test.ts": 29146, - "apps/webapp/test/runsReplicationService.part9.test.ts": 8107, - "apps/webapp/test/runsRepository.part1.test.ts": 31337, - "apps/webapp/test/runsRepository.part2.test.ts": 39180, - "apps/webapp/test/runsRepository.part3.test.ts": 30925, - "apps/webapp/test/runsRepository.part4.test.ts": 36000, - "apps/webapp/test/runsRepository.readthrough.test.ts": 48205, + "apps/webapp/test/runsReplicationService.part1.test.ts": 30406, + "apps/webapp/test/runsReplicationService.part10.test.ts": 18427, + "apps/webapp/test/runsReplicationService.part2.test.ts": 26828, + "apps/webapp/test/runsReplicationService.part3.test.ts": 15207, + "apps/webapp/test/runsReplicationService.part4.test.ts": 41894, + "apps/webapp/test/runsReplicationService.part5.test.ts": 10525, + "apps/webapp/test/runsReplicationService.part6.test.ts": 15181, + "apps/webapp/test/runsReplicationService.part7.test.ts": 71460, + "apps/webapp/test/runsReplicationService.part8.test.ts": 30321, + "apps/webapp/test/runsReplicationService.part9.test.ts": 15987, + "apps/webapp/test/runsRepository.part1.test.ts": 28851, + "apps/webapp/test/runsRepository.part2.test.ts": 28205, + "apps/webapp/test/runsRepository.part3.test.ts": 23041, + "apps/webapp/test/runsRepository.part4.test.ts": 25019, + "apps/webapp/test/runsRepository.readthrough.test.ts": 34731, "apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts": 5349, - "apps/webapp/test/runsRepositoryCpres.test.ts": 14258, - "apps/webapp/test/runsRepositoryCursor.test.ts": 49050, + "apps/webapp/test/runsRepositoryCpres.test.ts": 7188, + "apps/webapp/test/runsRepositoryCursor.test.ts": 33200, "apps/webapp/test/safeEnvironmentLog.test.ts": 1, "apps/webapp/test/safeIntegrationLog.test.ts": 2, "apps/webapp/test/safeRequestLogContext.test.ts": 4, @@ -451,19 +451,19 @@ "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 4, "apps/webapp/test/scheduleTimings.test.ts": 1506, "apps/webapp/test/scheduleWindow.test.ts": 31, - "apps/webapp/test/schedulesPutEnvScoping.test.ts": 15522, + "apps/webapp/test/schedulesPutEnvScoping.test.ts": 11109, "apps/webapp/test/selectBestEnvironment.test.ts": 1, "apps/webapp/test/sentryRequestIsolation.test.ts": 65, "apps/webapp/test/sentryTenantContext.test.ts": 4, "apps/webapp/test/sentryTraceContext.server.test.ts": 9, - "apps/webapp/test/services.controlPlane.readthrough.test.ts": 21682, + "apps/webapp/test/services.controlPlane.readthrough.test.ts": 7217, "apps/webapp/test/services/organizationAccessToken.test.ts": 6, "apps/webapp/test/services/personalAccessToken.test.ts": 7, - "apps/webapp/test/sessionDuration.test.ts": 16345, + "apps/webapp/test/sessionDuration.test.ts": 10855, "apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts": 13790, "apps/webapp/test/sessions.readthrough.test.ts": 9480, - "apps/webapp/test/sessionsReplicationService.test.ts": 22809, - "apps/webapp/test/setActiveOnTaskSchedule.test.ts": 7851, + "apps/webapp/test/sessionsReplicationService.test.ts": 19666, + "apps/webapp/test/setActiveOnTaskSchedule.test.ts": 9617, "apps/webapp/test/shouldRevalidateRunsList.test.ts": 2, "apps/webapp/test/slackErrorAlerts.test.ts": 1, "apps/webapp/test/slackOAuthResultLog.test.ts": 1, @@ -471,8 +471,8 @@ "apps/webapp/test/spanTraceRoutes.replicaLag.test.ts": 15558, "apps/webapp/test/streamBatchItemsAuthorization.test.ts": 4, "apps/webapp/test/streamLoader.controlPlane.test.ts": 14703, - "apps/webapp/test/syncDeclarativeSchedules.test.ts": 3536, - "apps/webapp/test/syncDeclarativeWebhooks.test.ts": 5631, + "apps/webapp/test/syncDeclarativeSchedules.test.ts": 10518, + "apps/webapp/test/syncDeclarativeWebhooks.test.ts": 17825, "apps/webapp/test/taskCodeSnippets.test.ts": 5, "apps/webapp/test/tenantContext.test.ts": 27, "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 3, @@ -482,8 +482,8 @@ "apps/webapp/test/timelineSpanEvents.test.ts": 3, "apps/webapp/test/traceExport.test.ts": 7, "apps/webapp/test/uatEnvironmentClaim.test.ts": 22, - "apps/webapp/test/updateMetadata.test.ts": 14977, - "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 14921, + "apps/webapp/test/updateMetadata.test.ts": 27671, + "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 8127, "apps/webapp/test/useTableSort.test.ts": 9, "apps/webapp/test/userActorEnvironmentScopeRouteBuilder.test.ts": 42, "apps/webapp/test/userActorPatOnlyBoundary.test.ts": 4565, @@ -491,25 +491,25 @@ "apps/webapp/test/userActorSourcePat.test.ts": 85, "apps/webapp/test/userActorTokenClaimsAndScopes.test.ts": 3360, "apps/webapp/test/utils/timezones.test.ts": 27, - "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 15923, - "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 16427, - "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 19703, + "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 8003, + "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 8169, + "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 8186, "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 17785, "apps/webapp/test/validateGitBranchName.test.ts": 5, "apps/webapp/test/vercelUrls.test.ts": 4, "apps/webapp/test/verifyDeploymentImage.test.ts": 621, "apps/webapp/test/viewAsUser.test.ts": 5, - "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 15766, + "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 8169, "apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts": 7980, "apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts": 17659, - "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 10384, + "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 10934, "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 16964, "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 17630, "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 15593, "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 17406, - "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 22650, + "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 31202, "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 9618, - "apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts": 29510, + "apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts": 8877, "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 12113, "apps/webapp/test/webhookErrorAlerts.test.ts": 4, "apps/webapp/test/workerGroupAccess.test.ts": 2, From d9cd57d7ae76065b4955a4245f59133a2e62117e Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 16:21:29 +0100 Subject: [PATCH 03/15] perf(ci): split webapp e2e tests across runners --- .github/workflows/e2e-webapp.yml | 9 ++++++++- apps/webapp/vitest.e2e.config.ts | 2 ++ test-timings.json | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml index ad7abea0383..e8f778d75b9 100644 --- a/.github/workflows/e2e-webapp.yml +++ b/.github/workflows/e2e-webapp.yml @@ -16,8 +16,15 @@ jobs: name: "๐Ÿงช E2E Tests: Webapp" runs-on: warp-ubuntu-latest-x64-16x timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2] + shardTotal: [2] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + SHARD_INDEX: ${{ matrix.shardIndex }} + SHARD_TOTAL: ${{ matrix.shardTotal }} steps: - name: ๐Ÿ”ง Disable IPv6 run: | @@ -130,6 +137,6 @@ jobs: run: cd apps/webapp && pnpm exec playwright install chromium - name: ๐Ÿงช Run Webapp E2E Tests - run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.config.ts --reporter=default + run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.config.ts --reporter=default --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} env: WEBAPP_TEST_VERBOSE: "1" diff --git a/apps/webapp/vitest.e2e.config.ts b/apps/webapp/vitest.e2e.config.ts index 905c4ac6f22..9cbf79c59a1 100644 --- a/apps/webapp/vitest.e2e.config.ts +++ b/apps/webapp/vitest.e2e.config.ts @@ -1,8 +1,10 @@ import { defineConfig } from "vitest/config"; +import { DurationShardingSequencer } from "@internal/testcontainers/sequencer"; import tsconfigPaths from "vite-tsconfig-paths"; export default defineConfig({ test: { + sequence: { sequencer: DurationShardingSequencer }, include: ["test/**/*.e2e.test.ts"], globals: true, pool: "forks", diff --git a/test-timings.json b/test-timings.json index 13406562eee..9397c0c667a 100644 --- a/test-timings.json +++ b/test-timings.json @@ -108,6 +108,7 @@ "apps/webapp/test/activitySeries.server.test.ts": 6, "apps/webapp/test/additionalApiKeyIssuance.test.ts": 3, "apps/webapp/test/aiTitleRateLimiter.test.ts": 55, + "apps/webapp/test/api-auth.e2e.test.ts": 20090, "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 94856, "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 174422, "apps/webapp/test/apiAuthActorClaim.test.ts": 4, @@ -279,6 +280,7 @@ "apps/webapp/test/getDeploymentImageRef.test.ts": 6, "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 466, "apps/webapp/test/googleEmailVerification.test.ts": 2, + "apps/webapp/test/healthcheck-require-plugins.e2e.test.ts": 37215, "apps/webapp/test/httpErrors.test.ts": 2, "apps/webapp/test/idempotencyDedupResidency.test.ts": 7600, "apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts": 2, @@ -459,7 +461,12 @@ "apps/webapp/test/services.controlPlane.readthrough.test.ts": 7217, "apps/webapp/test/services/organizationAccessToken.test.ts": 6, "apps/webapp/test/services/personalAccessToken.test.ts": 7, + "apps/webapp/test/session-agent.e2e.test.ts": 73920, + "apps/webapp/test/session-stream.browser.e2e.test.ts": 20470, + "apps/webapp/test/session-stream.e2e.test.ts": 30404, "apps/webapp/test/sessionDuration.test.ts": 10855, + "apps/webapp/test/sessionRunStreamsBackend.e2e.test.ts": 20771, + "apps/webapp/test/sessionRunStreamsPerOrgBasin.e2e.test.ts": 20639, "apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts": 13790, "apps/webapp/test/sessions.readthrough.test.ts": 9480, "apps/webapp/test/sessionsReplicationService.test.ts": 19666, From e0136cd13eb0baf362aadaf62db369fb55e8eb0d Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 16:58:24 +0100 Subject: [PATCH 04/15] perf(run-engine,redis-worker): stop leaking shutdown resources --- .../run-engine/src/engine/index.ts | 58 ++++++++--- .../src/engine/tests/shutdown.test.ts | 98 +++++++++++++++++++ packages/redis-worker/src/worker.test.ts | 69 ++++++++++++- packages/redis-worker/src/worker.ts | 22 +++-- 4 files changed, 227 insertions(+), 20 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/tests/shutdown.test.ts diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 2f7447af713..588c3a147d2 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -114,6 +114,7 @@ export class RunEngine { private repairSnapshotTimeoutMs: number; private batchQueue: BatchQueue; private workerQueueObserverAbortController?: AbortController; + private quitPromise?: Promise; prisma: PrismaClient; readOnlyPrisma: PrismaReplicaClient; @@ -2312,26 +2313,57 @@ export class RunEngine { } } - async quit() { - try { - this.workerQueueObserverAbortController?.abort(); + quit(): Promise { + this.quitPromise ??= this.#quit(); + return this.quitPromise; + } - await this.runQueue.quit(); - await this.worker.stop(); - await this.ttlWorker.stop(); - await this.runLock.quit(); + async #quit(): Promise { + this.workerQueueObserverAbortController?.abort(); - // This is just a failsafe - await this.runLockRedis.quit(); + // Stop resources that actively process work before closing support resources they may use. + const processingResults = await Promise.allSettled([ + this.runQueue.quit(), + this.worker.stop(), + this.ttlWorker.stop(), + this.batchQueue.close(), + ]); + this.#logShutdownFailures( + ["runQueue.quit", "worker.stop", "ttlWorker.stop", "batchQueue.close"], + processingResults + ); - await this.batchQueue.close(); + const supportResults = await Promise.allSettled([ + this.runLock.quit(), + this.debounceSystem.quit(), + ]); + this.#logShutdownFailures(["runLock.quit", "debounceSystem.quit"], supportResults); - await this.debounceSystem.quit(); - } catch (_error) { - // Best-effort shutdown; ignore quit/close errors. + // RunLocker/Redlock owns this client and normally closes it. Do not send a second QUIT, + // but force-disconnect if Redlock failed to leave the connection in its terminal state. + if (this.runLockRedis.status !== "end") { + try { + this.runLockRedis.disconnect(); + } catch (error) { + this.logger.error("RunEngine shutdown operation failed", { + operation: "runLockRedis.disconnect", + error, + }); + } } } + #logShutdownFailures(operations: string[], results: PromiseSettledResult[]): void { + results.forEach((result, index) => { + if (result.status === "rejected") { + this.logger.error("RunEngine shutdown operation failed", { + operation: operations[index], + error: result.reason, + }); + } + }); + } + async repairEnvironment(environment: AuthenticatedEnvironment, dryRun: boolean) { const runIds = await this.runQueue.getCurrentConcurrencyOfEnvironment(environment); diff --git a/internal-packages/run-engine/src/engine/tests/shutdown.test.ts b/internal-packages/run-engine/src/engine/tests/shutdown.test.ts new file mode 100644 index 00000000000..d12e73aee48 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/shutdown.test.ts @@ -0,0 +1,98 @@ +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { containerTestWithIsolatedRedisNoClickhouse } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; + +async function connectedClientCount(redis: Redis): Promise { + const clientsInfo = await redis.info("clients"); + const match = clientsInfo.match(/^connected_clients:(\d+)$/m); + + if (!match) { + throw new Error("Redis INFO clients response did not include connected_clients"); + } + + return Number(match[1]); +} + +function engineOptions(redisOptions: RedisOptions) { + // Keep caches and disabled consumers lazy so every connection opened by this test belongs to a + // shutdown resource. The run-lock client remains eager to exercise Redlock's ownership of it. + const lazyRedisOptions = { ...redisOptions, lazyConnect: true }; + + return { + worker: { + disabled: true, + redis: lazyRedisOptions, + workers: 1, + tasksPerWorker: 1, + pollIntervalMs: 10, + immediatePollIntervalMs: 10, + shutdownTimeoutMs: 30_000, + }, + queue: { + redis: lazyRedisOptions, + masterQueueConsumersDisabled: true, + ttlSystem: { disabled: true }, + logLevel: "error" as const, + }, + runLock: { redis: redisOptions }, + cache: { redis: lazyRedisOptions }, + debounce: { redis: lazyRedisOptions }, + batchQueue: { + redis: lazyRedisOptions, + consumerEnabled: false, + }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("run-engine-shutdown-test", "0.0.0"), + logger: new Logger("run-engine-shutdown-test", "error"), + }; +} + +describe("RunEngine.quit", () => { + containerTestWithIsolatedRedisNoClickhouse( + "is concurrency-safe, repeatable, and returns Redis connections to baseline", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + await prisma.$queryRaw`SELECT 1`; + + const observer = createRedisClient(redisOptions); + await observer.ping(); + const baselineConnections = await connectedClientCount(observer); + const engine = new RunEngine({ prisma, ...engineOptions(redisOptions) }); + + try { + await expect + .poll(() => connectedClientCount(observer)) + .toBeGreaterThan(baselineConnections); + + const firstQuit = engine.quit(); + const concurrentQuit = engine.quit(); + expect(concurrentQuit).toBe(firstQuit); + + await Promise.all([firstQuit, concurrentQuit, engine.quit()]); + + const repeatedQuit = engine.quit(); + expect(repeatedQuit).toBe(firstQuit); + await repeatedQuit; + + await expect.poll(() => connectedClientCount(observer)).toBe(baselineConnections); + } finally { + await engine.quit(); + await observer.quit(); + } + } + ); +}); diff --git a/packages/redis-worker/src/worker.test.ts b/packages/redis-worker/src/worker.test.ts index f5659f3795b..16d4ffe5476 100644 --- a/packages/redis-worker/src/worker.test.ts +++ b/packages/redis-worker/src/worker.test.ts @@ -4,7 +4,22 @@ import { describe } from "node:test"; import { expect } from "vitest"; import { z } from "zod"; import { Worker } from "./worker.js"; -import { createRedisClient } from "@internal/redis"; +import { createRedisClient, type Redis } from "@internal/redis"; + +async function connectedClientCount(redis: Redis): Promise { + const clientsInfo = await redis.info("clients"); + const match = clientsInfo.match(/^connected_clients:(\d+)$/m); + + if (!match) { + throw new Error("Redis INFO clients response did not include connected_clients"); + } + + return Number(match[1]); +} + +function activeTimeoutCount(): number { + return process.getActiveResourcesInfo().filter((resource) => resource === "Timeout").length; +} describe("Worker", () => { redisTest("Process items that don't throw", { timeout: 30_000 }, async ({ redisContainer }) => { @@ -549,6 +564,58 @@ describe("Worker", () => { } ); + redisTest( + "clears its shutdown deadline and closes Redis connections after a prompt stop", + { timeout: 30_000 }, + async ({ redisContainer }) => { + const redisOptions = { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }; + const observer = createRedisClient(redisOptions); + await observer.ping(); + + const baselineConnections = await connectedClientCount(observer); + const baselineTimeouts = activeTimeoutCount(); + const worker = new Worker({ + name: "shutdown-lifecycle-worker", + redisOptions, + catalog: { + testJob: { + schema: z.object({ value: z.number() }), + visibilityTimeoutMs: 5000, + }, + }, + jobs: { + testJob: async () => {}, + }, + concurrency: { workers: 1, tasksPerWorker: 1 }, + pollIntervalMs: 10, + immediatePollIntervalMs: 10, + shutdownTimeoutMs: 30_000, + logger: new Logger("shutdown-lifecycle-test", "error"), + }).start(); + + try { + await expect + .poll(() => connectedClientCount(observer)) + .toBeGreaterThan(baselineConnections); + + // Let the worker enter its polling loop so the loop, rather than the deadline, wins shutdown. + await new Promise((resolve) => setTimeout(resolve, 50)); + await worker.stop(); + + await expect.poll(() => connectedClientCount(observer)).toBe(baselineConnections); + await new Promise((resolve) => setImmediate(resolve)); + expect(activeTimeoutCount()).toBeLessThanOrEqual(baselineTimeouts); + } finally { + await worker.stop(); + await observer.quit(); + } + } + ); + redisTest( "Should allow cancelling a job before it's enqueued, but only if the enqueue.cancellationKey is provided", { timeout: 30_000 }, diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts index 64268a1c9e1..0db651a5427 100644 --- a/packages/redis-worker/src/worker.ts +++ b/packages/redis-worker/src/worker.ts @@ -1198,16 +1198,26 @@ class Worker { this.isShuttingDown = true; this.logger.log("Shutting down worker loops...", { signal }); - // Wait for all worker loops to finish. - await Promise.race([ - Promise.all(this.workerLoops), - Worker.delay(this.shutdownTimeoutMs).then(() => { + // Wait for all worker loops to finish, retaining ownership of the deadline timer so the + // losing timeout cannot keep the process alive after a prompt shutdown. + let shutdownDeadline: ReturnType | undefined; + const deadlinePromise = new Promise((resolve) => { + shutdownDeadline = setTimeout(() => { this.logger.error("Worker shutdown timed out", { signal, shutdownTimeoutMs: this.shutdownTimeoutMs, }); - }), - ]); + resolve(); + }, this.shutdownTimeoutMs); + }); + + try { + await Promise.race([Promise.all(this.workerLoops), deadlinePromise]); + } finally { + if (shutdownDeadline) { + clearTimeout(shutdownDeadline); + } + } await this.subscriber?.unsubscribe(); await this.subscriber?.quit(); From ff426755724fb4c25edecff85d3da9e121e75ad8 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 16:59:39 +0100 Subject: [PATCH 05/15] test(webapp): avoid repeated replication setup --- .../apiRunListPresenter.readthrough.test.ts | 142 +++++++ apps/webapp/test/apiRunListPresenter.test.ts | 395 ++++-------------- .../helpers/apiRunListPresenterTestHelpers.ts | 178 ++++++++ 3 files changed, 391 insertions(+), 324 deletions(-) create mode 100644 apps/webapp/test/apiRunListPresenter.readthrough.test.ts create mode 100644 apps/webapp/test/helpers/apiRunListPresenterTestHelpers.ts diff --git a/apps/webapp/test/apiRunListPresenter.readthrough.test.ts b/apps/webapp/test/apiRunListPresenter.readthrough.test.ts new file mode 100644 index 00000000000..5cd306cf3f9 --- /dev/null +++ b/apps/webapp/test/apiRunListPresenter.readthrough.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, vi } from "vitest"; + +// The presenter graph imports `~/db.server` singletons even though the asserted reads use explicit +// clients. These lazy proxies delegate every access to the real per-test Postgres containers. +const legacyReplicaHolder = vi.hoisted(() => ({ client: undefined as any })); +const newClientHolder = vi.hoisted(() => ({ client: undefined as any })); +const clickhouseHolder = vi.hoisted(() => ({ client: undefined as any })); + +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { + getClickhouseForOrganization: async () => { + if (!clickhouseHolder.client) { + throw new Error("clickhouseHolder.client not set for this test"); + } + return clickhouseHolder.client; + }, + }, +})); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_target, property) { + if (!holder.client) { + throw new Error(`${label} not set for this test`); + } + return holder.client[property]; + }, + } + ); + const replicaProxy = lazyProxy(legacyReplicaHolder, "legacyReplicaHolder.client"); + const newProxy = lazyProxy(newClientHolder, "newClientHolder.client"); + + return { + prisma: replicaProxy, + $replica: replicaProxy, + runOpsNewPrisma: newProxy, + runOpsNewReplica: newProxy, + runOpsLegacyPrisma: replicaProxy, + runOpsLegacyReplica: replicaProxy, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +import { createPostgresContainer, replicationContainerTest } from "@internal/testcontainers"; +import { PrismaClient } from "@trigger.dev/database"; +import { setTimeout } from "node:timers/promises"; +import { CURRENT_API_VERSION } from "~/api/versions"; +import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server"; +import { createRun, mirrorParents, seedParents } from "./helpers/apiRunListPresenterTestHelpers"; +import { setupClickhouseReplication } from "./utils/replicationUtils"; + +vi.setConfig({ testTimeout: 90_000 }); + +describe("ApiRunListPresenter public /runs routed read-through", () => { + replicationContainerTest( + "public payload lists run-ops rows served via the routed store (NEW + legacy union)", + async ({ clickhouseContainer, redisOptions, postgresContainer, prisma, network }) => { + const { clickhouse } = await setupClickhouseReplication({ + prisma, + databaseUrl: postgresContainer.getConnectionUri(), + clickhouseUrl: clickhouseContainer.getConnectionUrl(), + redisOptions, + }); + + const { url: newUrl } = await createPostgresContainer(network, { + imageTag: "docker.io/postgres:17", + }); + const prismaNew = new PrismaClient({ datasources: { db: { url: newUrl } } }); + legacyReplicaHolder.client = prisma; + clickhouseHolder.client = clickhouse; + newClientHolder.client = prismaNew; + + try { + const ctx = await seedParents(prisma, "hydrate"); + await mirrorParents(prismaNew, ctx, "hydrate"); + + // PG14 is the logical-replication source, so ClickHouse receives the complete ID set. + const legacyOnlyA = await createRun(prisma, ctx, { friendlyId: "run_legacyA" }); + const legacyOnlyB = await createRun(prisma, ctx, { friendlyId: "run_legacyB" }); + const migratedA = await createRun(prisma, ctx, { friendlyId: "run_newA" }); + const migratedB = await createRun(prisma, ctx, { friendlyId: "run_newB" }); + + // The routed PG17 rows use distinguishing values that prove NEW hydration won. + await createRun(prismaNew, ctx, { + friendlyId: "run_newA", + taskIdentifier: "my-task-NEW", + }); + await createRun(prismaNew, ctx, { + friendlyId: "run_newB", + taskIdentifier: "my-task-NEW", + }); + await prismaNew.taskRun.update({ + where: { friendlyId: "run_newA" }, + data: { id: migratedA.id }, + }); + await prismaNew.taskRun.update({ + where: { friendlyId: "run_newB" }, + data: { id: migratedB.id }, + }); + + await setTimeout(1500); + + const presenter = new ApiRunListPresenter(prisma, prisma, { + newClient: prismaNew, + legacyReplica: prisma, + splitEnabled: true, + }); + + const result = await presenter.call( + { id: ctx.projectId }, + { "page[size]": 10 } as any, + CURRENT_API_VERSION, + { id: ctx.environmentId, organizationId: ctx.organizationId } + ); + + const expectedFriendlyIds = [ + { id: migratedA.id, friendlyId: "run_newA" }, + { id: migratedB.id, friendlyId: "run_newB" }, + { id: legacyOnlyA.id, friendlyId: "run_legacyA" }, + { id: legacyOnlyB.id, friendlyId: "run_legacyB" }, + ] + .sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0)) + .map((run) => run.friendlyId); + expect(result.data.map((run) => run.id)).toEqual(expectedFriendlyIds); + + const migratedRow = result.data.find((run) => run.id === "run_newA"); + expect(migratedRow?.taskIdentifier).toBe("my-task-NEW"); + expect(migratedRow?.taskKind).toBe("STANDARD"); + expect(result.data.find((run) => run.id === "run_legacyA")?.taskIdentifier).toBe("my-task"); + + expect(result.pagination).toHaveProperty("next"); + expect(result.pagination).toHaveProperty("previous"); + } finally { + await prismaNew.$disconnect(); + } + } + ); +}); diff --git a/apps/webapp/test/apiRunListPresenter.test.ts b/apps/webapp/test/apiRunListPresenter.test.ts index bc9826461b9..a9f04ae5255 100644 --- a/apps/webapp/test/apiRunListPresenter.test.ts +++ b/apps/webapp/test/apiRunListPresenter.test.ts @@ -1,20 +1,11 @@ import { describe, expect, vi } from "vitest"; -// The presenter graph imports `~/v3/runStore.server` (via RunsRepository) which imports -// `~/db.server` at load, and the presenter itself reaches `~/db.server`'s `$replica` singleton -// through `findDisplayableEnvironment` and `getTaskIdentifiers`. Stub the module so those -// singleton reads resolve. This is the ONLY mock โ€” the DB is NEVER mocked; the proxy delegates -// to the per-test REAL legacy (PG14) container so the env-lookup + task-identifier reads hit a -// real database. Everything asserted runs against real containers. Mirrors -// nextRunListPresenter.readthrough.test.ts. +// The presenter graph imports `~/db.server` singletons even though these tests pass explicit real +// clients. The proxies keep those singleton reads on the current warm Postgres fixture. const legacyReplicaHolder = vi.hoisted(() => ({ client: undefined as any })); const newClientHolder = vi.hoisted(() => ({ client: undefined as any })); -// `ApiRunListPresenter` resolves its read ClickHouse internally via the `clickhouseFactory` -// singleton (which imports `~/env.server` and binds to a process-wide default client). Stub the -// instance module so `getClickhouseForOrganization` returns the per-test container's ClickHouse -// handle (set by each test before calling). This is a module-resolution shim โ€” the ClickHouse is -// a REAL testcontainer, never mocked โ€” mirroring the `~/db.server` stub below. const clickhouseHolder = vi.hoisted(() => ({ client: undefined as any })); + vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ clickhouseFactory: { getClickhouseForOrganization: async () => { @@ -25,22 +16,24 @@ vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ }, }, })); + vi.mock("~/db.server", async () => { const { Prisma } = await import("@trigger.dev/database"); const lazyProxy = (holder: { client: any }, label: string) => new Proxy( {}, { - get(_t, prop) { + get(_target, property) { if (!holder.client) { throw new Error(`${label} not set for this test`); } - return holder.client[prop]; + return holder.client[property]; }, } ); const replicaProxy = lazyProxy(legacyReplicaHolder, "legacyReplicaHolder.client"); const newProxy = lazyProxy(newClientHolder, "newClientHolder.client"); + return { prisma: replicaProxy, $replica: replicaProxy, @@ -52,347 +45,101 @@ vi.mock("~/db.server", async () => { }; }); -import { createPostgresContainer, replicationContainerTest } from "@internal/testcontainers"; -import { PrismaClient } from "@trigger.dev/database"; -import { setTimeout } from "node:timers/promises"; +import { ClickHouse } from "@internal/clickhouse"; +import { containerTest } from "@internal/testcontainers"; import { CURRENT_API_VERSION } from "~/api/versions"; import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server"; -import { setupClickhouseReplication } from "./utils/replicationUtils"; +import { + addEnvironment, + createRun, + insertTaskRunV2Rows, + seedParents, +} from "./helpers/apiRunListPresenterTestHelpers"; vi.setConfig({ testTimeout: 90_000 }); -type SeedContext = { - organizationId: string; - projectId: string; - environmentId: string; - environmentSlug: string; -}; - -/** - * Creates the org/project/env parents on a single prisma client. TaskRun FKs require these to - * exist on every DB a run lives on, so identical parents (same ids) are seeded on both the - * legacy (PG14) and new (PG17) databases. - */ -async function seedParents( - prisma: PrismaClient, - slug: string, - envSlug = `env-${slug}` -): Promise { - const organization = await prisma.organization.create({ - data: { title: `org-${slug}`, slug: `org-${slug}` }, +function setupClients(prisma: unknown, clickhouseUrl: string): ClickHouse { + const clickhouse = new ClickHouse({ + url: clickhouseUrl, + name: "api-run-list-presenter-test", + compression: { request: true }, }); - const project = await prisma.project.create({ - data: { - name: `proj-${slug}`, - slug: `proj-${slug}`, - organizationId: organization.id, - externalRef: `proj-${slug}`, - }, - }); - const runtimeEnvironment = await prisma.runtimeEnvironment.create({ - data: { - slug: envSlug, - type: "DEVELOPMENT", - projectId: project.id, - organizationId: organization.id, - apiKey: `tr_dev_${slug}`, - pkApiKey: `pk_dev_${slug}`, - shortcode: `sc-${slug}`, - }, - }); - - return { - organizationId: organization.id, - projectId: project.id, - environmentId: runtimeEnvironment.id, - environmentSlug: runtimeEnvironment.slug, - }; -} - -/** Adds an extra RuntimeEnvironment (control-plane row) to an existing project. */ -async function addEnvironment( - prisma: PrismaClient, - ctx: SeedContext, - slug: string, - envSlug: string -): Promise { - const env = await prisma.runtimeEnvironment.create({ - data: { - slug: envSlug, - type: "STAGING", - projectId: ctx.projectId, - organizationId: ctx.organizationId, - apiKey: `tr_${envSlug}_${slug}`, - pkApiKey: `pk_${envSlug}_${slug}`, - shortcode: `sc-${envSlug}-${slug}`, - }, - }); - return env.id; + legacyReplicaHolder.client = prisma; + newClientHolder.client = prisma; + clickhouseHolder.client = clickhouse; + return clickhouse; } -/** Mirrors the org/project/env parents onto a second DB with the SAME ids. */ -async function mirrorParents(prisma: PrismaClient, ctx: SeedContext, slug: string): Promise { - await prisma.organization.create({ - data: { id: ctx.organizationId, title: `org-${slug}`, slug: `org-${slug}` }, - }); - await prisma.project.create({ - data: { - id: ctx.projectId, - name: `proj-${slug}`, - slug: `proj-${slug}`, - organizationId: ctx.organizationId, - externalRef: `proj-${slug}`, - }, - }); - await prisma.runtimeEnvironment.create({ - data: { - id: ctx.environmentId, - slug: ctx.environmentSlug, - type: "DEVELOPMENT", - projectId: ctx.projectId, - organizationId: ctx.organizationId, - apiKey: `tr_dev_${slug}_b`, - pkApiKey: `pk_dev_${slug}_b`, - shortcode: `sc-${slug}-b`, - }, - }); -} - -async function createRun( - prisma: PrismaClient, - ctx: SeedContext, - run: { - friendlyId: string; - taskIdentifier?: string; - status?: any; - runtimeEnvironmentId?: string; - } -) { - return prisma.taskRun.create({ - data: { - friendlyId: run.friendlyId, - taskIdentifier: run.taskIdentifier ?? "my-task", - status: run.status ?? "PENDING", - payload: JSON.stringify({ foo: run.friendlyId }), - traceId: run.friendlyId, - spanId: run.friendlyId, - queue: "test", - runTags: [], - runtimeEnvironmentId: run.runtimeEnvironmentId ?? ctx.environmentId, - projectId: ctx.projectId, - organizationId: ctx.organizationId, - environmentType: "DEVELOPMENT", - engine: "V2", - }, - }); -} - -describe("ApiRunListPresenter public /runs list (PG14 legacy + PG17 new)", () => { - // Public list serves run-ops rows through the routed store. The - // forwarded readThroughDeps thread the dual-DB union into NextRunListPresenter; the public - // payload (`{ data, pagination }`) must list the NEW โˆช legacy union, proving the public API - // surfaces routed run-ops rows. The migrated/straggler rows (run_newA/run_newB) live on BOTH - // DBs with the same id + friendlyId but a DISTINGUISHING taskIdentifier ("my-task-NEW" on PG17), - // so a row served from the threaded newClient is identifiable in the public payload. - replicationContainerTest( - "public payload lists run-ops rows served via the routed store (NEW + legacy union)", - async ({ clickhouseContainer, redisOptions, postgresContainer, prisma, network }) => { - const { clickhouse } = await setupClickhouseReplication({ - prisma, - databaseUrl: postgresContainer.getConnectionUri(), - clickhouseUrl: clickhouseContainer.getConnectionUrl(), - redisOptions, - }); - - const { url: newUrl } = await createPostgresContainer(network, { - imageTag: "docker.io/postgres:17", - }); - const prismaNew = new PrismaClient({ datasources: { db: { url: newUrl } } }); - legacyReplicaHolder.client = prisma; - clickhouseHolder.client = clickhouse; - // The routed store's default known-migrated probe reads `runOpsNewPrisma` -> PG17. - newClientHolder.client = prismaNew; - - try { - const ctx = await seedParents(prisma, "hydrate"); - await mirrorParents(prismaNew, ctx, "hydrate"); - - // All four runs land on PG14 (legacy + replication source -> CH gets the full id-set). - const legacyOnlyA = await createRun(prisma, ctx, { friendlyId: "run_legacyA" }); - const legacyOnlyB = await createRun(prisma, ctx, { friendlyId: "run_legacyB" }); - const migratedA = await createRun(prisma, ctx, { friendlyId: "run_newA" }); - const migratedB = await createRun(prisma, ctx, { friendlyId: "run_newB" }); - - // The two "migrated" runs also live on NEW (authoritative during retention), same ids + - // friendlyIds, but a DISTINGUISHING taskIdentifier so a row served from PG17 is - // identifiable in the public payload. - await createRun(prismaNew, ctx, { friendlyId: "run_newA", taskIdentifier: "my-task-NEW" }); - await createRun(prismaNew, ctx, { friendlyId: "run_newB", taskIdentifier: "my-task-NEW" }); - await prismaNew.taskRun.update({ - where: { friendlyId: "run_newA" }, - data: { id: migratedA.id }, - }); - await prismaNew.taskRun.update({ - where: { friendlyId: "run_newB" }, - data: { id: migratedB.id }, - }); - - // Wait for CH replication so the id-set page is non-empty. - await setTimeout(1500); - - const presenter = new ApiRunListPresenter(prisma, prisma, { - newClient: prismaNew, - legacyReplica: prisma, - splitEnabled: true, - }); - - const result = await presenter.call( - { id: ctx.projectId }, - { "page[size]": 10 } as any, - CURRENT_API_VERSION, - { id: ctx.environmentId, organizationId: ctx.organizationId } - ); - - // The public payload lists runs by `id` = `run.friendlyId`, id-desc ordered. - const expectedFriendlyIds = [ - { id: migratedA.id, friendlyId: "run_newA" }, - { id: migratedB.id, friendlyId: "run_newB" }, - { id: legacyOnlyA.id, friendlyId: "run_legacyA" }, - { id: legacyOnlyB.id, friendlyId: "run_legacyB" }, - ] - .sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0)) - .map((r) => r.friendlyId); - expect(result.data.map((r) => r.id)).toEqual(expectedFriendlyIds); - - // The migrated rows must carry the PG17-only taskIdentifier โ€” only possible if the public - // path hydrated them through the threaded newClient (PG17). taskKind falls back to STANDARD. - const migratedRow = result.data.find((r) => r.id === "run_newA"); - expect(migratedRow?.taskIdentifier).toBe("my-task-NEW"); - expect(migratedRow?.taskKind).toBe("STANDARD"); - // The legacy-only rows surface from PG14, proving the legacyReplica is also exercised. - expect(result.data.find((r) => r.id === "run_legacyA")?.taskIdentifier).toBe("my-task"); - - // Pagination shape is present. - expect(result.pagination).toHaveProperty("next"); - expect(result.pagination).toHaveProperty("previous"); - } finally { - await prismaNew.$disconnect(); - } - } - ); - - // Genuinely-empty env returns { data: [], pagination } without error. Exercises the - // empty-state probe beneath NextRunListPresenter (no rows on either DB; empty CH page). - replicationContainerTest( +describe("ApiRunListPresenter public /runs list", () => { + containerTest( "genuinely-empty env returns { data: [], pagination } without error", - async ({ clickhouseContainer, redisOptions, postgresContainer, prisma, network }) => { - const { clickhouse } = await setupClickhouseReplication({ - prisma, - databaseUrl: postgresContainer.getConnectionUri(), - clickhouseUrl: clickhouseContainer.getConnectionUrl(), - redisOptions, - }); - - const { url: newUrl } = await createPostgresContainer(network, { - imageTag: "docker.io/postgres:17", + async ({ clickhouseContainer, prisma }) => { + setupClients(prisma, clickhouseContainer.getConnectionUrl()); + const ctx = await seedParents(prisma, "empty"); + + // Keep the split/read-through branch active while both real clients point at the empty DB. + const presenter = new ApiRunListPresenter(prisma, prisma, { + newClient: prisma, + legacyReplica: prisma, + splitEnabled: true, }); - const prismaNew = new PrismaClient({ datasources: { db: { url: newUrl } } }); - legacyReplicaHolder.client = prisma; - clickhouseHolder.client = clickhouse; - - try { - const ctx = await seedParents(prisma, "empty"); - await mirrorParents(prismaNew, ctx, "empty"); - const presenter = new ApiRunListPresenter(prisma, prisma, { - newClient: prismaNew, - legacyReplica: prisma, - splitEnabled: true, - }); - - const result = await presenter.call( - { id: ctx.projectId }, - { "page[size]": 10 } as any, - CURRENT_API_VERSION, - { id: ctx.environmentId, organizationId: ctx.organizationId } - ); + const result = await presenter.call( + { id: ctx.projectId }, + { "page[size]": 10 } as any, + CURRENT_API_VERSION, + { id: ctx.environmentId, organizationId: ctx.organizationId } + ); - expect(result.data).toEqual([]); - expect(result.pagination).toHaveProperty("next"); - expect(result.pagination).toHaveProperty("previous"); - } finally { - await prismaNew.$disconnect(); - } + expect(result.data).toEqual([]); + expect(result.pagination).toHaveProperty("next"); + expect(result.pagination).toHaveProperty("previous"); } ); - // Env scoping unchanged: the control-plane runtimeEnvironment.findMany lookup - // resolves the requested env via the `_replica` handle (NOT routed), with the 4th `environment` - // arg omitted to force that branch. Result is scoped to the requested env only. - replicationContainerTest( + containerTest( "env scoping resolves via the control-plane _replica handle (filter[env], 4th arg omitted)", - async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { - const { clickhouse } = await setupClickhouseReplication({ - prisma, - databaseUrl: postgresContainer.getConnectionUri(), - clickhouseUrl: clickhouseContainer.getConnectionUrl(), - redisOptions, - }); - - legacyReplicaHolder.client = prisma; - clickhouseHolder.client = clickhouse; - + async ({ clickhouseContainer, prisma }) => { + const clickhouse = setupClients(prisma, clickhouseContainer.getConnectionUrl()); const ctx = await seedParents(prisma, "scoping", "prod"); - const stagingEnvId = await addEnvironment(prisma, ctx, "scoping", "staging"); - - // Runs in prod only; a run in staging must NOT surface when filter[env]=prod. - await createRun(prisma, ctx, { friendlyId: "run_prod1" }); - await createRun(prisma, ctx, { friendlyId: "run_prod2" }); - await createRun(prisma, ctx, { - friendlyId: "run_staging", - runtimeEnvironmentId: stagingEnvId, - }); - - await setTimeout(1500); + const stagingEnvironmentId = await addEnvironment(prisma, ctx, "scoping", "staging"); + + // The Postgres rows exercise real hydration; matching ClickHouse rows provide the list IDs. + const runs = await Promise.all([ + createRun(prisma, ctx, { friendlyId: "run_prod1" }), + createRun(prisma, ctx, { friendlyId: "run_prod2" }), + createRun(prisma, ctx, { + friendlyId: "run_staging", + runtimeEnvironmentId: stagingEnvironmentId, + }), + ]); + await insertTaskRunV2Rows(clickhouse, runs); - // Single-handle passthrough; the env lookup runs on `_replica` (= prisma) via findMany. const presenter = new ApiRunListPresenter(prisma, prisma); - // 4th `environment` arg OMITTED -> forces the runtimeEnvironment.findMany branch. + // Omitting the fourth argument forces the control-plane runtimeEnvironment.findMany branch. const result = await presenter.call( { id: ctx.projectId }, { "page[size]": 10, "filter[env]": ["prod"] } as any, CURRENT_API_VERSION ); - // Scoped to the resolved prod env only. - expect(result.data.map((r) => r.id).sort()).toEqual(["run_prod1", "run_prod2"]); + expect(result.data.map((run) => run.id).sort()).toEqual(["run_prod1", "run_prod2"]); } ); - // Passthrough (single-DB): two-arg-style construction (no readThroughDeps) -> - // NextRunListPresenter receives undefined deps -> byte-identical single-DB path. The public - // { data, pagination } shape is unchanged. - replicationContainerTest( + containerTest( "single-DB passthrough: no readThroughDeps lists the seeded runs unchanged", - async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { - const { clickhouse } = await setupClickhouseReplication({ - prisma, - databaseUrl: postgresContainer.getConnectionUri(), - clickhouseUrl: clickhouseContainer.getConnectionUrl(), - redisOptions, - }); - - legacyReplicaHolder.client = prisma; - clickhouseHolder.client = clickhouse; - + async ({ clickhouseContainer, prisma }) => { + const clickhouse = setupClients(prisma, clickhouseContainer.getConnectionUrl()); const ctx = await seedParents(prisma, "passthrough"); - await createRun(prisma, ctx, { friendlyId: "run_pt1" }); - await createRun(prisma, ctx, { friendlyId: "run_pt2" }); - - await setTimeout(1500); + const runs = await Promise.all([ + createRun(prisma, ctx, { friendlyId: "run_pt1" }), + createRun(prisma, ctx, { friendlyId: "run_pt2" }), + ]); + await insertTaskRunV2Rows(clickhouse, runs); - // No readThroughDeps -> passthrough, exactly as the routes construct it today. + // No readThroughDeps preserves the single-database path used by existing callers. const presenter = new ApiRunListPresenter(prisma, prisma); const result = await presenter.call( @@ -402,7 +149,7 @@ describe("ApiRunListPresenter public /runs list (PG14 legacy + PG17 new)", () => { id: ctx.environmentId, organizationId: ctx.organizationId } ); - expect(result.data.map((r) => r.id).sort()).toEqual(["run_pt1", "run_pt2"]); + expect(result.data.map((run) => run.id).sort()).toEqual(["run_pt1", "run_pt2"]); expect(result).toHaveProperty("pagination"); expect(result.pagination).toHaveProperty("next"); expect(result.pagination).toHaveProperty("previous"); diff --git a/apps/webapp/test/helpers/apiRunListPresenterTestHelpers.ts b/apps/webapp/test/helpers/apiRunListPresenterTestHelpers.ts new file mode 100644 index 00000000000..e97b569ec65 --- /dev/null +++ b/apps/webapp/test/helpers/apiRunListPresenterTestHelpers.ts @@ -0,0 +1,178 @@ +import type { ClickHouse, TaskRunV2 } from "@internal/clickhouse"; +import type { PrismaClient, TaskRun, TaskRunStatus } from "@trigger.dev/database"; +import { z } from "zod"; + +export type SeedContext = { + organizationId: string; + projectId: string; + environmentId: string; + environmentSlug: string; +}; + +/** Creates the org/project/environment parents needed by TaskRun foreign keys. */ +export async function seedParents( + prisma: PrismaClient, + slug: string, + envSlug = `env-${slug}` +): Promise { + const organization = await prisma.organization.create({ + data: { title: `org-${slug}`, slug: `org-${slug}` }, + }); + const project = await prisma.project.create({ + data: { + name: `proj-${slug}`, + slug: `proj-${slug}`, + organizationId: organization.id, + externalRef: `proj-${slug}`, + }, + }); + const runtimeEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: envSlug, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slug}`, + pkApiKey: `pk_dev_${slug}`, + shortcode: `sc-${slug}`, + }, + }); + + return { + organizationId: organization.id, + projectId: project.id, + environmentId: runtimeEnvironment.id, + environmentSlug: runtimeEnvironment.slug, + }; +} + +/** Adds another control-plane environment to an existing project. */ +export async function addEnvironment( + prisma: PrismaClient, + ctx: SeedContext, + slug: string, + envSlug: string +): Promise { + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: envSlug, + type: "STAGING", + projectId: ctx.projectId, + organizationId: ctx.organizationId, + apiKey: `tr_${envSlug}_${slug}`, + pkApiKey: `pk_${envSlug}_${slug}`, + shortcode: `sc-${envSlug}-${slug}`, + }, + }); + + return environment.id; +} + +/** Mirrors the parents onto another database with the same IDs. */ +export async function mirrorParents( + prisma: PrismaClient, + ctx: SeedContext, + slug: string +): Promise { + await prisma.organization.create({ + data: { id: ctx.organizationId, title: `org-${slug}`, slug: `org-${slug}` }, + }); + await prisma.project.create({ + data: { + id: ctx.projectId, + name: `proj-${slug}`, + slug: `proj-${slug}`, + organizationId: ctx.organizationId, + externalRef: `proj-${slug}`, + }, + }); + await prisma.runtimeEnvironment.create({ + data: { + id: ctx.environmentId, + slug: ctx.environmentSlug, + type: "DEVELOPMENT", + projectId: ctx.projectId, + organizationId: ctx.organizationId, + apiKey: `tr_dev_${slug}_b`, + pkApiKey: `pk_dev_${slug}_b`, + shortcode: `sc-${slug}-b`, + }, + }); +} + +export async function createRun( + prisma: PrismaClient, + ctx: SeedContext, + run: { + friendlyId: string; + taskIdentifier?: string; + status?: TaskRunStatus; + runtimeEnvironmentId?: string; + } +): Promise { + return prisma.taskRun.create({ + data: { + friendlyId: run.friendlyId, + taskIdentifier: run.taskIdentifier ?? "my-task", + status: run.status ?? "PENDING", + payload: JSON.stringify({ foo: run.friendlyId }), + traceId: run.friendlyId, + spanId: run.friendlyId, + queue: "test", + runTags: [], + runtimeEnvironmentId: run.runtimeEnvironmentId ?? ctx.environmentId, + projectId: ctx.projectId, + organizationId: ctx.organizationId, + environmentType: "DEVELOPMENT", + engine: "V2", + }, + }); +} + +/** Inserts the ClickHouse list-index rows synchronously, without logical replication. */ +export async function insertTaskRunV2Rows(clickhouse: ClickHouse, runs: TaskRun[]): Promise { + const insert = clickhouse.writer.insert({ + name: "insertApiRunListPresenterTaskRuns", + table: "trigger_dev.task_runs_v2", + schema: z.any(), + settings: { async_insert: 0, enable_json_type: 1, type_json_skip_duplicated_paths: 1 }, + }); + + const rows: TaskRunV2[] = runs.map((run) => ({ + environment_id: run.runtimeEnvironmentId, + organization_id: run.organizationId ?? "", + project_id: run.projectId, + run_id: run.id, + friendly_id: run.friendlyId, + updated_at: run.updatedAt.getTime(), + created_at: run.createdAt.getTime(), + status: run.status, + environment_type: run.environmentType ?? "DEVELOPMENT", + attempt: run.attemptNumber ?? 1, + engine: run.engine, + task_identifier: run.taskIdentifier, + queue: run.queue, + schedule_id: "", + batch_id: "", + task_version: run.taskVersion ?? "", + sdk_version: run.sdkVersion ?? "", + cli_version: run.cliVersion ?? "", + machine_preset: run.machinePreset ?? "", + root_run_id: "", + parent_run_id: "", + span_id: run.spanId, + trace_id: run.traceId, + idempotency_key: run.idempotencyKey ?? "", + expiration_ttl: run.ttl ?? "", + tags: run.runTags, + worker_queue: run.workerQueue, + region: run.region ?? "", + _version: String(run.updatedAt.getTime()), + _is_deleted: 0, + })); + + const [error] = await insert(rows); + if (error) { + throw error; + } +} From f3b636c062d8b7e4219091cb67dd5180dcffe14b Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 17:01:02 +0100 Subject: [PATCH 06/15] test(webapp): reduce repeated container setup --- .../triggerTask.server.nullBytes.test.ts | 37 +---- .../envConcurrencyLimitPause.server.test.ts | 151 ++---------------- ...ConcurrencyLimitPauseDirect.server.test.ts | 71 ++++++++ ...oncurrencyLimitPauseService.server.test.ts | 56 +++++++ .../envConcurrencyLimitPauseTestHelpers.ts | 94 +++++++++++ 5 files changed, 240 insertions(+), 169 deletions(-) create mode 100644 apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts create mode 100644 apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts create mode 100644 apps/webapp/test/helpers/envConcurrencyLimitPauseTestHelpers.ts diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts index 9612f103e4b..29bb51ff310 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts @@ -50,7 +50,7 @@ function buildService(engine: any, prisma: any) { describe("RunEngineTriggerTaskService null-byte sanitization", () => { containerTest( - "strips a NUL from idempotencyKeyOptions.key so the jsonb insert does not 22P05", + "sanitizes NUL-containing idempotency and debounce keys before the jsonb insert", async ({ prisma, redisOptions }) => { const engine = buildEngine(prisma, redisOptions); @@ -59,41 +59,13 @@ describe("RunEngineTriggerTaskService null-byte sanitization", () => { const service = buildService(engine, prisma); const result = await service.call({ - taskId: "nul-idem-task", + taskId: "nul-keys-task", environment, body: { - payload: { kind: "idem" }, + payload: { kind: "nul-keys" }, options: { idempotencyKey: "a".repeat(64), idempotencyKeyOptions: { key: `acme${NUL}inc`, scope: "run" }, - }, - }, - }); - assertNonNullable(result); - - const row = await prisma.taskRun.findUniqueOrThrow({ where: { id: result.run.id } }); - expect(row.idempotencyKeyOptions).toEqual({ key: "acmeinc", scope: "run" }); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "strips a NUL from debounce.key so the jsonb insert does not 22P05", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const service = buildService(engine, prisma); - - const result = await service.call({ - taskId: "nul-debounce-task", - environment, - body: { - payload: { kind: "debounce" }, - options: { debounce: { key: `grp${NUL}1`, delay: "1s" }, }, }, @@ -101,7 +73,8 @@ describe("RunEngineTriggerTaskService null-byte sanitization", () => { assertNonNullable(result); const row = await prisma.taskRun.findUniqueOrThrow({ where: { id: result.run.id } }); - expect((row.debounce as { key: string }).key).toBe("grp1"); + expect(row.idempotencyKeyOptions).toEqual({ key: "acmeinc", scope: "run" }); + expect(row.debounce).toMatchObject({ key: "grp1", delay: "1s" }); } finally { await engine.quit(); } diff --git a/apps/webapp/test/envConcurrencyLimitPause.server.test.ts b/apps/webapp/test/envConcurrencyLimitPause.server.test.ts index 7bb67c81345..ea78de33b21 100644 --- a/apps/webapp/test/envConcurrencyLimitPause.server.test.ts +++ b/apps/webapp/test/envConcurrencyLimitPause.server.test.ts @@ -1,15 +1,14 @@ -import { RunEngine } from "@internal/run-engine"; import { containerTest } from "@internal/testcontainers"; -import { trace } from "@opentelemetry/api"; import type { PrismaClient } from "@trigger.dev/database"; import type { RedisOptions } from "ioredis"; import { describe, expect, onTestFinished, vi } from "vitest"; -import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { - createRuntimeEnvironment, - createTestOrgProjectWithMember, - uniqueId, -} from "./fixtures/environmentVariablesFixtures"; + authEnv, + createEnvConcurrencyLimitPauseTestEngine, + type EnvConcurrencyLimitPauseTestEngine, + loadEnvConcurrencyLimitPauseServices, + seedProductionEnv, +} from "./helpers/envConcurrencyLimitPauseTestHelpers"; vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); @@ -18,100 +17,33 @@ vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); // real RunQueue state, so put a real RunEngine - built on the test's own Redis container - back // behind the singleton. No test here uses the no-op default. const { engineHolder } = vi.hoisted(() => ({ - engineHolder: { current: undefined as any }, + engineHolder: { current: undefined as EnvConcurrencyLimitPauseTestEngine | undefined }, })); vi.mock("~/v3/runEngine.server", () => ({ - engine: new Proxy({} as Record, { - get: (_target, prop) => engineHolder.current?.[prop as string], + engine: new Proxy({} as Record, { + get: (_target, prop) => + engineHolder.current ? Reflect.get(engineHolder.current, prop) : undefined, }), })); function useEngine(prisma: PrismaClient, redisOptions: RedisOptions) { - const engine = new RunEngine({ - prisma, - worker: { redis: redisOptions, disabled: true }, - queue: { redis: redisOptions, masterQueueConsumersDisabled: true }, - runLock: { redis: redisOptions }, - machines: { - defaultMachine: "small-1x", - machines: { - "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, - }, - baseCostInCents: 0.0001, - }, - tracer: trace.getTracer("test", "0.0.0"), - }); - + const engine = createEnvConcurrencyLimitPauseTestEngine(prisma, redisOptions); engineHolder.current = engine; onTestFinished(async () => { engineHolder.current = undefined; await engine.quit(); }); - return engine; } -// The import chain reaches module-level singletons that throw at load time when -// REDIS_HOST/REDIS_PORT are unset (autoIncrementCounter via triggerTaskV1), so the env must point -// at the redis container BEFORE the modules are imported. Hence dynamic imports; vitest runs each -// file in its own fork, so the env mutation cannot leak into other suites. -async function loadServices(redisOptions: RedisOptions) { - process.env.REDIS_HOST = redisOptions.host; - process.env.REDIS_PORT = String(redisOptions.port); - process.env.REDIS_TLS_DISABLED = "true"; - const [{ updateEnvConcurrencyLimits }, { PauseEnvironmentService }, runtimeEnvironment] = - await Promise.all([ - import("~/v3/runQueue.server"), - import("~/v3/services/pauseEnvironment.server"), - import("~/models/runtimeEnvironment.server"), - ]); - return { - updateEnvConcurrencyLimits, - PauseEnvironmentService, - authIncludeBase: runtimeEnvironment.authIncludeBase, - toAuthenticated: runtimeEnvironment.toAuthenticated, - }; -} - -type Loaded = Awaited>; - -async function authEnv( - loaded: Loaded, - prisma: PrismaClient, - environmentId: string -): Promise { - const row = await prisma.runtimeEnvironment.findFirstOrThrow({ - where: { id: environmentId }, - include: loaded.authIncludeBase, - }); - return loaded.toAuthenticated(row); -} - -async function seedProductionEnv(prisma: PrismaClient, maximumConcurrencyLimit: number) { - const { organization, project } = await createTestOrgProjectWithMember(prisma); - const environment = await createRuntimeEnvironment(prisma, { - projectId: project.id, - organizationId: organization.id, - type: "PRODUCTION", - slug: uniqueId("prod"), - }); - - await prisma.runtimeEnvironment.update({ - where: { id: environment.id }, - data: { maximumConcurrencyLimit }, - }); - - return { organization, project, environment }; -} - // An unset RunQueue limit reads back as the engine default (10), so neither the 0 nor the 17 // assertions below can pass just because a push never happened. -describe("updateEnvConcurrencyLimits", () => { +describe("updateEnvConcurrencyLimits with stale environments", () => { containerTest( "clamps to 0 when the environment is paused, even though the caller's copy says otherwise", async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); const engine = useEngine(prisma, redisOptions); const { environment } = await seedProductionEnv(prisma, 17); @@ -132,25 +64,10 @@ describe("updateEnvConcurrencyLimits", () => { } ); - containerTest( - "pushes the real limit for a running environment", - async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); - const engine = useEngine(prisma, redisOptions); - - const { environment } = await seedProductionEnv(prisma, 17); - const env = await authEnv(loaded, prisma, environment.id); - - await loaded.updateEnvConcurrencyLimits(env, undefined, prisma); - - expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(17); - } - ); - containerTest( "restores the real limit when the environment was resumed while the request was in flight", async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); const engine = useEngine(prisma, redisOptions); const { environment } = await seedProductionEnv(prisma, 17); @@ -174,44 +91,4 @@ describe("updateEnvConcurrencyLimits", () => { expect(await engine.runQueue.getEnvConcurrencyLimit(whilePaused)).toBe(17); } ); - - containerTest( - "an explicit limit wins over the stored pause state", - async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); - const engine = useEngine(prisma, redisOptions); - - const { environment } = await seedProductionEnv(prisma, 17); - await prisma.runtimeEnvironment.update({ - where: { id: environment.id }, - data: { paused: true }, - }); - const env = await authEnv(loaded, prisma, environment.id); - - // How billing-limit converge restores a limit as it unpauses: the caller decides, no read. - await loaded.updateEnvConcurrencyLimits(env, 9, prisma); - - expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(9); - } - ); - - containerTest( - "a pause writes 0 and a resume restores the limit", - async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); - const engine = useEngine(prisma, redisOptions); - - const { environment } = await seedProductionEnv(prisma, 17); - const service = new loaded.PauseEnvironmentService(prisma); - const env = await authEnv(loaded, prisma, environment.id); - - expect(await service.call(env, "paused")).toEqual({ success: true, state: "paused" }); - expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(0); - - // The service holds an environment read before its own resume update, so `env.paused` is - // stale here too. - expect(await service.call(env, "resumed")).toEqual({ success: true, state: "resumed" }); - expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(17); - } - ); }); diff --git a/apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts b/apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts new file mode 100644 index 00000000000..0662d49fcb8 --- /dev/null +++ b/apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts @@ -0,0 +1,71 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "ioredis"; +import { describe, expect, onTestFinished, vi } from "vitest"; +import { + authEnv, + createEnvConcurrencyLimitPauseTestEngine, + type EnvConcurrencyLimitPauseTestEngine, + loadEnvConcurrencyLimitPauseServices, + seedProductionEnv, +} from "./helpers/envConcurrencyLimitPauseTestHelpers"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const { engineHolder } = vi.hoisted(() => ({ + engineHolder: { current: undefined as EnvConcurrencyLimitPauseTestEngine | undefined }, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: new Proxy({} as Record, { + get: (_target, prop) => + engineHolder.current ? Reflect.get(engineHolder.current, prop) : undefined, + }), +})); + +function useEngine(prisma: PrismaClient, redisOptions: RedisOptions) { + const engine = createEnvConcurrencyLimitPauseTestEngine(prisma, redisOptions); + engineHolder.current = engine; + onTestFinished(async () => { + engineHolder.current = undefined; + await engine.quit(); + }); + return engine; +} + +describe("updateEnvConcurrencyLimits directly", () => { + containerTest( + "pushes the real limit for a running environment", + async ({ prisma, redisOptions }) => { + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); + const engine = useEngine(prisma, redisOptions); + + const { environment } = await seedProductionEnv(prisma, 17); + const env = await authEnv(loaded, prisma, environment.id); + + await loaded.updateEnvConcurrencyLimits(env, undefined, prisma); + + expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(17); + } + ); + + containerTest( + "an explicit limit wins over the stored pause state", + async ({ prisma, redisOptions }) => { + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); + const engine = useEngine(prisma, redisOptions); + + const { environment } = await seedProductionEnv(prisma, 17); + await prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { paused: true }, + }); + const env = await authEnv(loaded, prisma, environment.id); + + // How billing-limit converge restores a limit as it unpauses: the caller decides, no read. + await loaded.updateEnvConcurrencyLimits(env, 9, prisma); + + expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(9); + } + ); +}); diff --git a/apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts b/apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts new file mode 100644 index 00000000000..eb4570757ef --- /dev/null +++ b/apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts @@ -0,0 +1,56 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "ioredis"; +import { describe, expect, onTestFinished, vi } from "vitest"; +import { + authEnv, + createEnvConcurrencyLimitPauseTestEngine, + type EnvConcurrencyLimitPauseTestEngine, + loadEnvConcurrencyLimitPauseServices, + seedProductionEnv, +} from "./helpers/envConcurrencyLimitPauseTestHelpers"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const { engineHolder } = vi.hoisted(() => ({ + engineHolder: { current: undefined as EnvConcurrencyLimitPauseTestEngine | undefined }, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: new Proxy({} as Record, { + get: (_target, prop) => + engineHolder.current ? Reflect.get(engineHolder.current, prop) : undefined, + }), +})); + +function useEngine(prisma: PrismaClient, redisOptions: RedisOptions) { + const engine = createEnvConcurrencyLimitPauseTestEngine(prisma, redisOptions); + engineHolder.current = engine; + onTestFinished(async () => { + engineHolder.current = undefined; + await engine.quit(); + }); + return engine; +} + +describe("PauseEnvironmentService", () => { + containerTest( + "a pause writes 0 and a resume restores the limit", + async ({ prisma, redisOptions }) => { + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); + const engine = useEngine(prisma, redisOptions); + + const { environment } = await seedProductionEnv(prisma, 17); + const service = new loaded.PauseEnvironmentService(prisma); + const env = await authEnv(loaded, prisma, environment.id); + + expect(await service.call(env, "paused")).toEqual({ success: true, state: "paused" }); + expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(0); + + // The service holds an environment read before its own resume update, so `env.paused` is + // stale here too. + expect(await service.call(env, "resumed")).toEqual({ success: true, state: "resumed" }); + expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(17); + } + ); +}); diff --git a/apps/webapp/test/helpers/envConcurrencyLimitPauseTestHelpers.ts b/apps/webapp/test/helpers/envConcurrencyLimitPauseTestHelpers.ts new file mode 100644 index 00000000000..0edff7b6548 --- /dev/null +++ b/apps/webapp/test/helpers/envConcurrencyLimitPauseTestHelpers.ts @@ -0,0 +1,94 @@ +import { RunEngine } from "@internal/run-engine"; +import { trace } from "@opentelemetry/api"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "ioredis"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + createRuntimeEnvironment, + createTestOrgProjectWithMember, + uniqueId, +} from "../fixtures/environmentVariablesFixtures"; + +export type EnvConcurrencyLimitPauseTestEngine = RunEngine; + +export function createEnvConcurrencyLimitPauseTestEngine( + prisma: PrismaClient, + redisOptions: RedisOptions +) { + return new RunEngine({ + prisma, + worker: { redis: redisOptions, disabled: true }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + ttlSystem: { disabled: true }, + }, + batchQueue: { redis: redisOptions, consumerEnabled: false }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} + +// The import chain reaches module-level singletons that throw at load time when +// REDIS_HOST/REDIS_PORT are unset (autoIncrementCounter via triggerTaskV1), so the env must point +// at the redis container BEFORE the modules are imported. Vitest runs each file in its own fork, +// so the env mutation cannot leak into other suites. +export async function loadEnvConcurrencyLimitPauseServices(redisOptions: RedisOptions) { + process.env.REDIS_HOST = redisOptions.host; + process.env.REDIS_PORT = String(redisOptions.port); + process.env.REDIS_TLS_DISABLED = "true"; + const [{ updateEnvConcurrencyLimits }, { PauseEnvironmentService }, runtimeEnvironment] = + await Promise.all([ + import("~/v3/runQueue.server"), + import("~/v3/services/pauseEnvironment.server"), + import("~/models/runtimeEnvironment.server"), + ]); + + return { + updateEnvConcurrencyLimits, + PauseEnvironmentService, + authIncludeBase: runtimeEnvironment.authIncludeBase, + toAuthenticated: runtimeEnvironment.toAuthenticated, + }; +} + +export type EnvConcurrencyLimitPauseServices = Awaited< + ReturnType +>; + +export async function authEnv( + loaded: EnvConcurrencyLimitPauseServices, + prisma: PrismaClient, + environmentId: string +): Promise { + const row = await prisma.runtimeEnvironment.findFirstOrThrow({ + where: { id: environmentId }, + include: loaded.authIncludeBase, + }); + + return loaded.toAuthenticated(row); +} + +export async function seedProductionEnv(prisma: PrismaClient, maximumConcurrencyLimit: number) { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + await prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { maximumConcurrencyLimit }, + }); + + return { organization, project, environment }; +} From d0d555bc7ffac61a58dc3fd84851b2a328eeea60 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 17:07:13 +0100 Subject: [PATCH 07/15] test(webapp): split dashboard watch suites --- .../test/dashboardAgentWatches.batch.test.ts | 704 ++++ .../dashboardAgentWatches.delivery.test.ts | 809 ++++ .../dashboardAgentWatches.lifecycle.test.ts | 629 +++ .../test/dashboardAgentWatches.routes.test.ts | 731 ++++ .../test/dashboardAgentWatches.submit.test.ts | 617 +++ .../webapp/test/dashboardAgentWatches.test.ts | 3421 ----------------- .../dashboardAgentWatchesTestHelpers.ts | 219 ++ 7 files changed, 3709 insertions(+), 3421 deletions(-) create mode 100644 apps/webapp/test/dashboardAgentWatches.batch.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatches.delivery.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatches.routes.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatches.submit.test.ts delete mode 100644 apps/webapp/test/dashboardAgentWatches.test.ts create mode 100644 apps/webapp/test/helpers/dashboardAgentWatchesTestHelpers.ts diff --git a/apps/webapp/test/dashboardAgentWatches.batch.test.ts b/apps/webapp/test/dashboardAgentWatches.batch.test.ts new file mode 100644 index 00000000000..ba71db7bbd8 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.batch.test.ts @@ -0,0 +1,704 @@ +import { + armWatchBatch, + cancelWatch, + claimWatchBatchTick, + claimWatchDelivery, + createChat, + getWatch, + listActiveWatchesForBatch, + listWatchBatchGroupsToArm, + markWatchDelivered, + recordWatchCheck, + stopWatchBatch, + transitionWatchCondition, + WATCH_DELIVERY_CLAIM_STALE_MS, + type DashboardAgentDb, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import { + DashboardAgentWatchesTestHarness, + HEALTH, + RUN_START, + type DashboardAgentWatchesTestContext, + type Seeded, +} from "./helpers/dashboardAgentWatchesTestHelpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted( + (): DashboardAgentWatchesTestContext => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined, + triggered: [], + }) +); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => ctx.canAccess, +})); + +const SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.SESSION_SECRET = SESSION_SECRET; + +const { armDashboardAgentWatchBatch, createDashboardAgentWatch, watchBatchStaleMs } = + await import("~/services/dashboardAgentWatches.server"); +const { rearmDashboardAgentWatchBatches } = + await import("~/services/dashboardAgentWatchSweep.server"); +const { runWatchBatchCheck } = await import("~/services/dashboardAgentWatchBatch.server"); +const { signDashboardAgentWatchBatchToken, signDashboardAgentWatchToken } = + await import("~/services/dashboardAgentWatchToken.server"); +const { action: batchCheckAction } = + await import("~/routes/api.v1.dashboard-agent.watches.batch-check"); + +const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); +const boot = harness.boot.bind(harness); +const seed = harness.seed.bind(harness); +const authenticated = harness.authenticated.bind(harness); +const seedChat = harness.seedChat.bind(harness); +const fakeCheckDeps = harness.fakeCheckDeps.bind(harness); +const create = harness.create.bind(harness); + +beforeEach(() => harness.reset()); +afterEach(() => harness.close()); + +describe("the batch chain registry", () => { + postgresTest("arms one chain per group, and only one", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batcharm"); + const now = new Date(); + + const scheduled: Array<{ epoch: number; tick: number }> = []; + const arm = () => + armDashboardAgentWatchBatch({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + now, + deps: { + schedule: async (params) => + void scheduled.push({ epoch: params.epoch, tick: params.tick }), + }, + }); + + expect(await arm()).toEqual({ running: true }); + expect(scheduled).toEqual([{ epoch: 1, tick: 1 }]); + + expect(await arm()).toEqual({ running: true }); + expect(await arm()).toEqual({ running: true }); + expect(scheduled).toHaveLength(1); + }); + + postgresTest( + "a chain whose run died is re-armed on a fresh epoch, and the zombie claims nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchdead"); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + const scheduled: Array<{ epoch: number; tick: number }> = []; + const arm = (now: Date) => + armDashboardAgentWatchBatch({ + ...group, + now, + deps: { + schedule: async (params) => + void scheduled.push({ epoch: params.epoch, tick: params.tick }), + }, + }); + + const armedAt = new Date(); + await arm(armedAt); + expect( + await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 1 }) + ).toMatchObject({ epoch: 1, generation: 1 }); + + await arm(new Date(armedAt.getTime() + 60_000)); + expect(scheduled).toHaveLength(1); + + await arm(new Date(armedAt.getTime() + watchBatchStaleMs(5) + 60_000)); + expect(scheduled).toEqual([ + { epoch: 1, tick: 1 }, + { epoch: 2, tick: 1 }, + ]); + + expect(await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 2 })).toBe( + null + ); + expect( + await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 2, generation: 1 }) + ).toMatchObject({ epoch: 2, generation: 1 }); + } + ); + + postgresTest( + "a chain that couldn't be triggered is not left marked as running", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchfail"); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + expect( + await armDashboardAgentWatchBatch({ + ...group, + deps: { + schedule: async () => { + throw new Error("the trigger failed"); + }, + }, + }) + ).toEqual({ running: false }); + + const scheduled: number[] = []; + expect( + await armDashboardAgentWatchBatch({ + ...group, + deps: { schedule: async (params) => void scheduled.push(params.epoch) }, + }) + ).toEqual({ running: true }); + expect(scheduled).toEqual([2]); + } + ); + + postgresTest( + "the re-arm backstop finds groups with active watches and no chain", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchrearm"); + await seedChat(seeded); + const created = await create({ + seeded, + spec: HEALTH, + checkDeps: { readHealth: async () => null }, + }); + expect(created.ok).toBe(true); + + const groups = await listWatchBatchGroupsToArm(ctx.agentDb); + expect(groups).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + + const armed: Array<{ environmentId: string; cadenceMinutes: number }> = []; + expect( + await rearmDashboardAgentWatchBatches({ + configured: () => true, + arm: async (params) => { + armed.push({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + return { running: true }; + }, + }) + ).toEqual({ stale: 1, armed: 1, failed: 0 }); + expect(armed).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + + // The staleness window is the group's own cadence: a five-minute group goes stale 17 minutes later. + await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + staleBefore: new Date(), + }); + expect(await listWatchBatchGroupsToArm(ctx.agentDb)).toEqual([]); + expect( + await listWatchBatchGroupsToArm(ctx.agentDb, { + now: new Date(Date.now() + watchBatchStaleMs(5) + 60_000), + }) + ).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + } + ); + + postgresTest( + "groups are per environment and per cadence, never mixed", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchgroup"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + expect((await create({ seeded, chatId: "chat_1", spec: HEALTH })).ok).toBe(true); + expect((await create({ seeded, chatId: "chat_2", spec: RUN_START })).ok).toBe(true); + + const five = await listActiveWatchesForBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + }); + const one = await listActiveWatchesForBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 1, + }); + + expect(five.map((watch) => watch.chatId)).toEqual(["chat_1"]); + expect(one.map((watch) => watch.chatId)).toEqual(["chat_2"]); + expect( + (await listWatchBatchGroupsToArm(ctx.agentDb)).sort( + (a, b) => a.cadenceMinutes - b.cadenceMinutes + ) + ).toEqual([ + { environmentId: seeded.environment.id, cadenceMinutes: 1 }, + { environmentId: seeded.environment.id, cadenceMinutes: 5 }, + ]); + } + ); +}); + +describe("the batch check", () => { + async function healthGroup(seeded: Seeded, count = 3) { + const ids: string[] = []; + for (let index = 0; index < count; index++) { + const chatId = `chat_${index + 1}`; + await seedChat(seeded, chatId); + const created = await create({ + seeded, + chatId, + spec: HEALTH, + // `warn` keeps them all pending, so the group stays whole for the assertions below. + checkDeps: { readHealth: async () => ({ trustworthy: true, severity: "warn" }) }, + }); + if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); + ids.push(created.watchId); + } + return ids; + } + + async function otherUsersWatch(seeded: Seeded, prisma: PrismaClient) { + const user = await prisma.user.create({ + data: { + email: `other_${seeded.organization.slug}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: user.id, role: "MEMBER" }, + }); + await createChat(ctx.agentDb, { + id: "chat_other", + organizationId: seeded.organization.id, + userId: user.id, + }); + const created = await createDashboardAgentWatch({ + environment: authenticated(seeded), + userId: user.id, + chatId: "chat_other", + spec: HEALTH, + deps: { + configured: () => true, + checkDeps: () => + fakeCheckDeps({ readHealth: async () => ({ trustworthy: true, severity: "warn" }) }), + scheduleTick: async () => {}, + }, + }); + if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); + return { userId: user.id, watchId: created.watchId }; + } + + async function armChain(seeded: Seeded, cadenceMinutes = 5) { + const row = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes, + staleBefore: new Date(), + }); + if (!row) throw new Error("the chain wasn't armed"); + return row; + } + + postgresTest( + "authorizes once and loads the shared report once for the whole group", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchcheck"); + const ids = await healthGroup(seeded); + const chain = await armChain(seeded); + + let healthReads = 0; + let authorizations = 0; + + const response = await runWatchBatchCheck( + { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }, + { + authorize: async () => { + authorizations++; + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => + fakeCheckDeps({ + readHealth: async () => { + healthReads++; + return { trustworthy: true, severity: "warn" }; + }, + }), + } + ); + + expect(authorizations).toBe(1); + expect(healthReads).toBe(1); + + expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual([...ids].sort()); + expect(response.watches?.every((entry) => entry.result === "pending")).toBe(true); + expect(response.watches?.every((entry) => entry.tick === 1)).toBe(true); + expect(response.watches?.every((entry) => entry.token.length > 0)).toBe(true); + expect(response.continues).toBe(true); + expect(response.stale).toBeUndefined(); + + for (const id of ids) { + expect((await getWatch(ctx.agentDb, { id }))?.lastResult).toMatchObject({ + result: "pending", + final: false, + }); + } + } + ); + + postgresTest( + "authorizes each distinct user, so sharing readers never shares access", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchusers"); + await healthGroup(seeded, 2); + const other = await otherUsersWatch(seeded, prisma); + + const chain = await armChain(seeded); + const authorized: string[] = []; + + await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async (watch) => { + authorized.push(watch.userId); + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(authorized.sort()).toEqual([other.userId, seeded.user.id].sort()); + } + ); + + postgresTest( + "cancels a watch whose user lost access, and still answers for its neighbours", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchrevoked"); + const ids = await healthGroup(seeded, 2); + const chain = await armChain(seeded); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async () => ({ ok: false, reason: "access_revoked" }), + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(response.watches?.every((entry) => entry.code === "access_revoked")).toBe(true); + for (const id of ids) { + expect(await getWatch(ctx.agentDb, { id })).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + } + } + ); + + postgresTest( + "checks what is due, skips what isn't, and never skips a window boundary", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchdue"); + const [fresh, overdue, boundary] = await healthGroup(seeded, 3); + const chain = await armChain(seeded); + const now = new Date(); + + await recordWatchCheck(ctx.agentDb, { id: fresh!, lastCheckedAt: now }); + await recordWatchCheck(ctx.agentDb, { + id: overdue!, + lastCheckedAt: new Date(now.getTime() - 10 * 60_000), + }); + // `boundary`'s window closes before the next tick, so its final evaluation must still happen. + await recordWatchCheck(ctx.agentDb, { id: boundary!, lastCheckedAt: now }); + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() + interval '1 minute' where id = $1`, + boundary + ); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + now: () => now, + authorize: async () => ({ ok: true, environment: authenticated(seeded) }), + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual( + [boundary!, overdue!].sort() + ); + expect(response.continues).toBe(true); + } + ); + + postgresTest( + "a stale tick claims nothing and checks nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchstale"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch }; + expect((await runWatchBatchCheck({ ...group, tick: 1 })).stale).toBeUndefined(); + expect((await runWatchBatchCheck({ ...group, tick: 2 })).stale).toBeUndefined(); + + const late = await runWatchBatchCheck({ ...group, tick: 1 }); + expect(late).toEqual({ stale: true }); + + expect(await runWatchBatchCheck({ ...group, epoch: chain.epoch - 1, tick: 1 })).toEqual({ + stale: true, + }); + expect((await getWatch(ctx.agentDb, { id: ids[0]! }))?.status).toBe("active"); + } + ); + + postgresTest( + "stops the chain when the group's last watch is gone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchempty"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + await cancelWatch(ctx.agentDb, { id: ids[0]!, reason: "user" }); + + const response = await runWatchBatchCheck({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }); + + expect(response).toMatchObject({ watches: [], continues: false }); + + const rearmed = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + // Deliberately in the past: only a stopped chain can be re-armed this way. + staleBefore: new Date(Date.now() - 60 * 60_000), + }); + expect(rearmed).toMatchObject({ epoch: chain.epoch + 1, status: "running" }); + } + ); + + postgresTest( + "hands the group's owed wakes back for redelivery", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchowed"); + const ids = await healthGroup(seeded, 2); + const chain = await armChain(seeded); + + await transitionWatchCondition(ctx.agentDb, { + id: ids[0]!, + resolution: "condition_met", + lastResult: { verified: true }, + }); + + const response = await runWatchBatchCheck({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }); + + const owed = response.watches?.filter((entry) => entry.deliverOnly === true) ?? []; + expect(owed.map((entry) => entry.watchId)).toEqual([ids[0]!]); + expect(owed[0]?.tick).toBe(0); + expect( + response.watches?.filter((entry) => !entry.deliverOnly).map((entry) => entry.watchId) + ).toEqual([ids[1]!]); + } + ); + + postgresTest( + "keeps the chain alive while a wake is still owed, even with nothing left to watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchowedlast"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + await transitionWatchCondition(ctx.agentDb, { + id: ids[0]!, + resolution: "condition_met", + lastResult: { verified: true }, + }); + + const first = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 1 }); + expect(first.continues).toBe(true); + expect(first.watches?.map((entry) => entry.deliverOnly)).toEqual([true]); + + const claim = await claimWatchDelivery(ctx.agentDb, { + id: ids[0]!, + staleBefore: new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS), + }); + await markWatchDelivered(ctx.agentDb, { id: ids[0]!, claimId: claim!.claimId }); + + const second = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 2 }); + expect(second).toMatchObject({ watches: [], continues: false }); + expect(await stopWatchBatch(ctx.agentDb, { ...group, epoch: chain.epoch })).toBe(null); + } + ); + + postgresTest( + "one watch that throws mid-evaluation costs only that watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchthrow"); + const mine = await healthGroup(seeded, 2); + const theirs = await otherUsersWatch(seeded, prisma); + const chain = await armChain(seeded); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async (watch) => { + if (watch.userId === theirs.userId) throw new Error("the authorization query failed"); + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => fakeCheckDeps(), + concurrency: 1, + } + ); + + const byId = new Map(response.watches?.map((entry) => [entry.watchId, entry])); + expect(byId.get(theirs.watchId)).toMatchObject({ result: "unavailable" }); + expect((await getWatch(ctx.agentDb, { id: theirs.watchId }))?.status).toBe("active"); + for (const id of mine) { + expect(byId.get(id)).toMatchObject({ result: "pending" }); + } + } + ); +}); + +describe("the batch check endpoint's authorization", () => { + function batchRequest(body: unknown, token?: string) { + return new Request("https://app.trigger.dev/api/v1/dashboard-agent/watches/batch-check", { + method: "POST", + headers: { + ...(token ? { Authorization: `Bearer ${token}` } : {}), + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + } + + const batchToken = (environmentId: string, cadenceMinutes: number) => + signDashboardAgentWatchBatchToken(SESSION_SECRET, { + environmentId, + cadenceMinutes, + expiresAt: new Date(Date.now() + 60 * 60_000), + }); + + postgresTest("refuses a missing or bad token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const body = { environmentId: "env_1", cadenceMinutes: 5, epoch: 1, tick: 1 }; + + expect( + (await batchCheckAction({ request: batchRequest(body), params: {}, context: {} })).status + ).toBe(401); + const watchToken = await signDashboardAgentWatchToken(SESSION_SECRET, { + watchId: "watch_1", + expiresAt: new Date(Date.now() + 60 * 60_000), + }); + expect( + (await batchCheckAction({ request: batchRequest(body, watchToken), params: {}, context: {} })) + .status + ).toBe(401); + }); + + postgresTest( + "refuses a token minted for another group", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const token = await batchToken("env_1", 5); + + const wrongCadence = await batchCheckAction({ + request: batchRequest( + { environmentId: "env_1", cadenceMinutes: 15, epoch: 1, tick: 1 }, + token + ), + params: {}, + context: {}, + }); + expect(wrongCadence.status).toBe(403); + expect(await wrongCadence.json()).toMatchObject({ code: "group_mismatch" }); + + const wrongEnvironment = await batchCheckAction({ + request: batchRequest( + { environmentId: "env_2", cadenceMinutes: 5, epoch: 1, tick: 1 }, + token + ), + params: {}, + context: {}, + }); + expect(wrongEnvironment.status).toBe(403); + } + ); + + postgresTest( + "answers a group it does own, through the real registry", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchroute"); + const chain = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + staleBefore: new Date(), + }); + const token = await batchToken(seeded.environment.id, 5); + + const response = await batchCheckAction({ + request: batchRequest( + { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain!.epoch, + tick: 1, + }, + token + ), + params: {}, + context: {}, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ watches: [], continues: false }); + expect( + await stopWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain!.epoch, + }) + ).toBe(null); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.delivery.test.ts b/apps/webapp/test/dashboardAgentWatches.delivery.test.ts new file mode 100644 index 00000000000..7552d1667ec --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.delivery.test.ts @@ -0,0 +1,809 @@ +import { + appendChatMessageOnce, + cancelWatch, + chatExists, + claimWatchDelivery, + claimWatchTick, + countUnreadWatchWakes, + getChatMessages, + getWatch, + listActiveWatchesForChat, + listChatIdsWithUnreadWakes, + listRecentWatchWakes, + markWatchDelivered, + readWatchWakeFeed, + recordWatchCheck, + releaseWatchDelivery, + transitionWatchCondition, + WATCH_DELIVERY_CLAIM_STALE_MS, + type DashboardAgentDb, + type Watch, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; +import { + DashboardAgentWatchesTestHarness, + RUN_START, + type DashboardAgentWatchesTestContext, + type Seeded, +} from "./helpers/dashboardAgentWatchesTestHelpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted( + (): DashboardAgentWatchesTestContext => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined, + triggered: [], + }) +); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-tokens"; + +const { + cancelDashboardAgentWatch, + createDashboardAgentWatch, + deleteChatWithWatches, + listActiveWatchesForChats, +} = await import("~/services/dashboardAgentWatches.server"); +const { sweepDashboardAgentWatches, WATCH_DELIVERY_GRACE_MS, WATCH_EXPIRY_GRACE_MS } = + await import("~/services/dashboardAgentWatchSweep.server"); + +const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); +const boot = harness.boot.bind(harness); +const seed = harness.seed.bind(harness); +const authenticated = harness.authenticated.bind(harness); +const seedChat = harness.seedChat.bind(harness); +const runRow = harness.runRow.bind(harness); +const fakeCheckDeps = harness.fakeCheckDeps.bind(harness); +const create = harness.create.bind(harness); +const storedMessages = harness.storedMessages.bind(harness); + +beforeEach(() => harness.reset()); +afterEach(() => harness.close()); + +describe("the chat cascade and the list view", () => { + postgresTest( + "deleting a chat soft-deletes it and cancels its active watches in one call", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "cascade"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + + const mine = await create({ seeded, chatId: "chat_1" }); + const theirs = await create({ seeded, chatId: "chat_2" }); + expect(mine.ok && theirs.ok).toBe(true); + if (!mine.ok || !theirs.ok) return; + + expect( + await deleteChatWithWatches({ + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) + ).toEqual({ + deleted: true, + cancelledWatches: 1, + }); + + expect( + await chatExists(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) + ).toBe(false); + expect(await getWatch(ctx.agentDb, { id: mine.watchId })).toMatchObject({ + status: "cancelled", + cancelReason: "chat_deleted", + deliveryStatus: "not_required", + }); + expect(await getWatch(ctx.agentDb, { id: theirs.watchId })).toMatchObject({ + status: "active", + }); + } + ); + + postgresTest( + "a user's own cancel leaves one neutral line in the chat, and only one", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "usercancel"); + await seedChat(seeded, "chat_1"); + + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const cancel = () => + cancelDashboardAgentWatch({ + watchId: created.watchId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + + expect(await cancel()).toMatchObject({ + cancelled: true, + messages: [ + { + id: `watch-cancelled:${created.watchId}`, + role: "assistant", + parts: [{ type: "text", text: "Stopped watching run run_1." }], + }, + ], + }); + expect(await getWatch(ctx.agentDb, { id: created.watchId })).toMatchObject({ + status: "cancelled", + cancelReason: "user", + deliveryStatus: "not_required", + }); + expect(await storedMessages(seeded, "chat_1")).toMatchObject([ + { id: `watch-cancelled:${created.watchId}`, role: "assistant" }, + ]); + + // The row is no longer active, so the second cancel writes nothing at all. + expect(await cancel()).toEqual({ cancelled: false, messages: [] }); + expect(await storedMessages(seeded, "chat_1")).toHaveLength(1); + } + ); + + postgresTest( + "a chat delete cancels its watches without a line in the chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "silentcancel"); + await seedChat(seeded, "chat_1"); + + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + + await deleteChatWithWatches({ + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + + const rows = await ctx.prisma.$queryRawUnsafe<{ message_id: string }[]>( + `select message_id from trigger_dashboard_agent.chat_messages where chat_id = 'chat_1'` + ); + expect(rows).toEqual([]); + } + ); + + postgresTest( + "aggregates active watches per chat in one query", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "chips"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + + const a = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_1" } }); + const b = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_2" } }); + const c = await create({ seeded, chatId: "chat_2" }); + expect(a.ok && b.ok && c.ok).toBe(true); + + const byChat = await listActiveWatchesForChats({ + chatIds: ["chat_1", "chat_2", "chat_missing"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + expect(byChat.chat_1).toHaveLength(2); + expect(byChat.chat_2).toHaveLength(1); + expect(byChat.chat_missing).toBeUndefined(); + expect(byChat.chat_2![0]).toMatchObject({ + identity: "run_start:run_1", + status: "active", + kind: "run_start", + note: RUN_START.note, + }); + + if (a.ok) await cancelWatch(ctx.agentDb, { id: a.watchId, reason: "user" }); + if (b.ok) await cancelWatch(ctx.agentDb, { id: b.watchId, reason: "user" }); + expect( + ( + await listActiveWatchesForChats({ + chatIds: ["chat_1"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).chat_1 + ).toBeUndefined(); + } + ); + + postgresTest("returns nothing for an empty chat list", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + expect( + await listActiveWatchesForChats({ chatIds: [], organizationId: "org_x", userId: "user_x" }) + ).toEqual({}); + }); +}); + +describe("unread watch wakes", () => { + postgresTest( + "only signals a wake once its delivery landed", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "unread"); + await seedChat(seeded, "chat_1"); + + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const scope = { organizationId: seeded.organization.id, userId: seeded.user.id }; + const recent = { ...scope, deliveredAfter: new Date(Date.now() - 15 * 60 * 1000) }; + + if (!created.watching) throw new Error("expected a watch"); + await transitionWatchCondition(ctx.agentDb, { + id: created.watchId, + resolution: "condition_met", + }); + expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(0); + expect(await listRecentWatchWakes(ctx.agentDb, recent)).toEqual([]); + expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set()); + + await markWatchDelivered(ctx.agentDb, { id: created.watchId }); + expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(1); + expect(await listRecentWatchWakes(ctx.agentDb, recent)).toMatchObject([ + { watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }, + ]); + expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set(["chat_1"])); + + // The poll's single query answers both halves the same way. + expect(await readWatchWakeFeed(ctx.agentDb, recent)).toMatchObject({ + unreadWakes: 1, + wakes: [{ watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }], + }); + + // An unread wake from before the window still counts, but isn't narrated again. + expect( + await readWatchWakeFeed(ctx.agentDb, { + ...scope, + deliveredAfter: new Date(Date.now() + 60_000), + }) + ).toMatchObject({ unreadWakes: 1, wakes: [] }); + } + ); +}); + +describe("the watch sweep", () => { + async function overdueWatch(seeded: Seeded, chatId = "chat_1") { + const created = await create({ seeded, chatId }); + if (!created.ok) throw new Error("the watch wasn't created"); + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 hour' where id = $1`, + created.watchId + ); + return created.watchId; + } + + function sweepDeps(args: { + seeded: Seeded; + checkDeps?: Partial; + revoked?: boolean; + now?: Date; + failDelivery?: boolean; + delivered: string[]; + }) { + return { + now: () => args.now ?? new Date(), + checkDeps: () => fakeCheckDeps(args.checkDeps), + authorize: async () => + args.revoked + ? ({ ok: false, reason: "access_revoked" } as const) + : ({ ok: true, environment: authenticated(args.seeded) } as const), + deliver: async (watch: Watch) => { + if (args.failDelivery) throw new Error("the delivery couldn't be scheduled"); + args.delivered.push(watch.id); + }, + configured: () => true, + }; + } + + postgresTest( + "runs the final check on an overdue watch and fires it at the buzzer", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches( + sweepDeps({ + seeded, + delivered, + checkDeps: { + readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), + }, + }) + ); + + expect(result).toMatchObject({ overdue: 1, fired: 1, expired: 0, cancelled: 0, failed: 0 }); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "fired", + deliveryStatus: "pending", + }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "expires an overdue watch the check says hasn't happened, as verified", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); + + expect(result).toMatchObject({ overdue: 1, expired: 1, failed: 0 }); + const row = await getWatch(ctx.agentDb, { id: watchId }); + expect(row).toMatchObject({ status: "expired", deliveryStatus: "pending" }); + expect(row?.lastResult).toMatchObject({ verified: true, reason: "not_met_by_expiry" }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "cancels an overdue watch whose user lost access, and never wakes the chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches( + sweepDeps({ seeded, delivered, revoked: true }) + ); + + expect(result).toMatchObject({ overdue: 1, cancelled: 1, expired: 0, fired: 0, failed: 0 }); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "leaves a watch that is still inside its deadline alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); + + expect(result).toMatchObject({ overdue: 0, undelivered: 0 }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "recovers a wake the delivery lost, through the real query, exactly once", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + await expect( + sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, failDelivery: true })) + ).rejects.toThrow(/failed on 1 watches/); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "expired", + deliveryStatus: "pending", + }); + expect(delivered).toEqual([]); + + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const second = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + expect(second).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(delivered).toEqual([watchId]); + + await markWatchDelivered(ctx.agentDb, { id: watchId }); + const third = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + expect(third).toMatchObject({ undelivered: 0, redelivered: 0 }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "a deliverer that died mid-delivery is recovered, but a fresh claim is left alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] })); + + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_status = 'delivering', + delivery_claimed_at = now(), + last_checked_at = now() - interval '1 hour' + where id = $1`, + watchId + ); + expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] }))).toMatchObject({ + undelivered: 0, + }); + + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_claimed_at = now() - interval '1 hour' where id = $1`, + watchId + ); + const recovered: string[] = []; + expect( + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: recovered })) + ).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(recovered).toEqual([watchId]); + } + ); + + postgresTest( + "leaves nothing owed for a request the immediate check already answered", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + + const created = await create({ + seeded, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + expect(created.ok).toBe(true); + if (!created.ok || created.watching) throw new Error("expected a one-shot result"); + + const delivered: string[] = []; + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + + expect(result).toMatchObject({ overdue: 0, undelivered: 0, redelivered: 0 }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "finalizes overdue watches even with no agent to deliver to, and delivers once it's back", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const unconfigured = await sweepDashboardAgentWatches({ + ...sweepDeps({ seeded, delivered }), + configured: () => false, + }); + + expect(unconfigured).toMatchObject({ + overdue: 1, + expired: 1, + deliveryDeferred: 1, + undelivered: 0, + redelivered: 0, + failed: 0, + }); + expect(delivered).toEqual([]); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "expired", + deliveryStatus: "pending", + }); + + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const restored = await sweepDashboardAgentWatches( + sweepDeps({ seeded, delivered, now: later }) + ); + expect(restored).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "the expiry grace keeps the sweep off a watch the tick chain is still finishing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + // A second past the deadline, so the chain's own final check owns this window. + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 second' where id = $1`, + created.watchId + ); + const delivered: string[] = []; + expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered }))).toMatchObject({ + overdue: 0, + }); + + const later = new Date(Date.now() + WATCH_EXPIRY_GRACE_MS + 60_000); + expect( + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })) + ).toMatchObject({ overdue: 1, expired: 1 }); + } + ); +}); + +describe("the tick claim", () => { + postgresTest( + "claiming a generation is not an observation: only a recorded check stamps one", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const claimed = await claimWatchTick(ctx.agentDb, { id: created.watchId, generation: 1 }); + expect(claimed).toMatchObject({ tickCount: 1, lastCheckedAt: null, lastResult: null }); + + await recordWatchCheck(ctx.agentDb, { id: created.watchId, lastResult: { pending: 4 } }); + const row = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(row?.lastCheckedAt).toBeInstanceOf(Date); + expect(row?.lastResult).toMatchObject({ pending: 4 }); + expect(row?.tickCount).toBe(1); + } + ); +}); + +// The delivery claim's fencing token: a hung deliverer is taken over, so an unfenced release or mark would touch the new owner's claim. +describe("the delivery claim", () => { + async function firedWatch(seeded: Seeded) { + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) throw new Error("the watch wasn't created"); + const transitioned = await transitionWatchCondition(ctx.agentDb, { + id: created.watchId, + status: "fired", + lastResult: { result: "satisfied", facts: { verified: true } }, + }); + expect(transitioned).toMatchObject({ deliveryStatus: "pending" }); + return created.watchId; + } + + function staleBefore() { + return new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS); + } + + async function ageClaim(watchId: string) { + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_claimed_at = now() - interval '1 hour' where id = $1`, + watchId + ); + } + + postgresTest( + "a stale takeover makes the old owner's release a no-op, and the new owner delivers once", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-fence"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(a).not.toBeNull(); + if (!a) return; + + await ageClaim(watchId); + const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(b).not.toBeNull(); + if (!b) return; + expect(b.claimId).not.toBe(a.claimId); + + expect( + await releaseWatchDelivery(ctx.agentDb, { id: watchId, claimId: a.claimId }) + ).toBeNull(); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivering", + deliveryClaimId: b.claimId, + }); + + expect( + await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) + ).toBeNull(); + + expect( + await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) + ).toMatchObject({ deliveryStatus: "delivered" }); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId })).toBeNull(); + } + ); + + postgresTest( + "a late delivered-mark from the old owner completes nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-late"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(a).not.toBeNull(); + if (!a) return; + await ageClaim(watchId); + const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(b).not.toBeNull(); + if (!b) return; + + expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: a.claimId })).toBeNull(); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivering", + deliveredAt: null, + }); + + expect( + await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) + ).toMatchObject({ deliveryStatus: "delivered" }); + } + ); + + postgresTest( + "the inline path marks a pending delivery without a claim", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-inline"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivered", + }); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); + expect( + await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) + ).toBeNull(); + } + ); +}); + +describe("deleting a chat while a watch is being created", () => { + postgresTest("holds in both orders", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + + for (const deleteFirst of [true, false]) { + const chatId = `chat_${deleteFirst ? "del" : "add"}`; + await seedChat(seeded, chatId); + + const creating = () => create({ seeded, chatId }); + const deleting = () => + deleteChatWithWatches({ + chatId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + const [a, b] = deleteFirst + ? await Promise.all([deleting(), creating()]) + : await Promise.all([creating(), deleting()]); + expect(a).toBeDefined(); + expect(b).toBeDefined(); + + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId })).toEqual([]); + expect( + await chatExists(ctx.agentDb, { + chatId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) + ).toBe(false); + } + }); + + postgresTest( + "refuses a create against an already-deleted chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + await seedChat(seeded); + await deleteChatWithWatches({ + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + + expect(await create({ seeded })).toMatchObject({ ok: false, code: "chat_not_found" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); +}); + +describe("appendChatMessageOnce", () => { + postgresTest( + "appends in order without rewriting the transcript", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "append"); + await seedChat(seeded); + + const first = { id: "watch-card:watch_1", role: "assistant", parts: [] }; + const second = { id: "watch-card:watch_2", role: "assistant", parts: [] }; + + expect( + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + message: first, + }) + ).toBe(true); + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + message: second, + }); + + const messages = await getChatMessages(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + expect(messages).toEqual([first, second]); + } + ); + + postgresTest( + "appends nothing for a chat the caller doesn't own", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "append-owner"); + await seedChat(seeded); + + expect( + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: "user_someone_else", + organizationId: seeded.organization.id, + message: { id: "watch-card:watch_1", role: "assistant", parts: [] }, + }) + ).toBe(false); + + const messages = await getChatMessages(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + expect(messages).toEqual([]); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts b/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts new file mode 100644 index 00000000000..e54cb597406 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts @@ -0,0 +1,629 @@ +import { + getWatch, + listActiveWatchesForChat, + recordWatchCheck, + type DashboardAgentDb, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import { previousCheckFacts } from "~/services/dashboardAgentWatchChecks"; +import { + BACKLOG, + DashboardAgentWatchesTestHarness, + RUN_START, + readRunOnce, + type DashboardAgentWatchesTestContext, +} from "./helpers/dashboardAgentWatchesTestHelpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted( + (): DashboardAgentWatchesTestContext => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined, + triggered: [], + }) +); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => ctx.canAccess, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-tokens"; + +const { authorizeWatchEnvironment, createDashboardAgentWatch, listActiveWatchesForChats } = + await import("~/services/dashboardAgentWatches.server"); + +const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); +const boot = harness.boot.bind(harness); +const seed = harness.seed.bind(harness); +const seedChat = harness.seedChat.bind(harness); +const runRow = harness.runRow.bind(harness); +const create = harness.create.bind(harness); + +beforeEach(() => harness.reset()); +afterEach(() => harness.close()); + +describe("createDashboardAgentWatch", () => { + postgresTest( + "creates an active watch and schedules its first tick", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const scheduled: Array<{ watchId: string; token: string; tick: number }> = []; + const result = await create({ seeded, scheduled }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.status).toBe("active"); + expect(result.identity).toBe("run_start:run_1"); + expect(result.immediate).toBeUndefined(); + + expect(scheduled).toHaveLength(1); + expect(scheduled[0]!.watchId).toBe(result.watchId); + expect(scheduled[0]!.tick).toBe(1); + expect(scheduled[0]!.token.startsWith("tr_daw_")).toBe(true); + + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + expect(row).toMatchObject({ + status: "active", + deliveryStatus: "not_required", + environmentId: seeded.environment.id, + projectId: seeded.project.id, + organizationId: seeded.organization.id, + userId: seeded.user.id, + tickCount: 0, + investigateOnAttention: false, + projectRef: seeded.project.externalRef, + }); + } + ); + + postgresTest( + "records the investigate-on-attention consent when the caller asks for it", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ seeded, investigateOnAttention: true }); + + expect(result.ok).toBe(true); + if (!result.ok || !result.watching) return; + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + expect(row?.investigateOnAttention).toBe(true); + expect(result.identity).toBe("run_start:run_1"); + } + ); + + postgresTest( + "stamps a server-set `since` on an error_recurrence watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const before = Date.now(); + const result = await create({ + seeded, + spec: { + kind: "error_recurrence", + fingerprint: "fp_1", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it comes back", + }, + }); + expect(result.ok).toBe(true); + if (!result.ok) return; + + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + const since = (row?.spec as { since?: string } | undefined)?.since; + expect(since).toBeDefined(); + expect(new Date(since!).getTime()).toBeGreaterThanOrEqual(before - 1000); + } + ); + + postgresTest( + "answers with a one-shot result and writes no row when the condition already holds", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + let ticks = 0; + const result = await create({ + seeded, + checkDeps: { + readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), + }, + onSchedule: () => { + ticks += 1; + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok || result.watching) throw new Error("expected a one-shot result"); + expect(result.immediate.result).toBe("satisfied"); + expect(result.immediate.observed).toMatchObject({ kind: "run_start", started: true }); + expect(ticks).toBe(0); + + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + expect( + await listActiveWatchesForChats({ + chatIds: ["chat_1"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).toEqual({}); + } + ); + + postgresTest( + "answers with a one-shot result when the condition can no longer happen", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + checkDeps: { readRun: readRunOnce(runRow({ status: "QUEUED" })) }, + }); + + expect(result.ok).toBe(true); + if (!result.ok || result.watching) throw new Error("expected a one-shot result"); + expect(result.immediate.result).toBe("terminal_unsatisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses a duplicate before running the immediate check", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const first = await create({ seeded }); + expect(first.ok).toBe(true); + + let checks = 0; + const second = await create({ + seeded, + checkDeps: { + readRun: async () => { + checks += 1; + return runRow({ status: "EXECUTING", startedAt: new Date() }); + }, + }, + }); + + expect(second).toMatchObject({ ok: false, code: "duplicate" }); + expect(checks).toBe(1); + } + ); + + postgresTest( + "cancels the row silently when the first tick can't be scheduled", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + onSchedule: () => { + throw new Error("no agent project"); + }, + }); + + expect(result).toMatchObject({ ok: false, code: "internal" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + const rows = await ctx.prisma.$queryRawUnsafe< + { status: string; cancel_reason: string; delivery_status: string }[] + >( + `select status, cancel_reason, delivery_status + from trigger_dashboard_agent.watches where chat_id = 'chat_1'` + ); + expect(rows).toMatchObject([ + { + status: "cancelled", + cancel_reason: "scheduling_failed", + delivery_status: "not_required", + }, + ]); + } + ); + + postgresTest( + "rejects a target that doesn't exist, writing nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: BACKLOG, + checkDeps: { queueExists: async () => false }, + }); + + expect(result).toMatchObject({ ok: false, code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "dedups the same condition and allows it in another environment", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const first = await create({ seeded }); + expect(first.ok).toBe(true); + + const second = await create({ seeded }); + expect(second).toMatchObject({ ok: false, code: "duplicate" }); + if (!second.ok && first.ok) expect(second.existingId).toBe(first.watchId); + + const otherEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "stg", + type: "STAGING", + projectId: seeded.project.id, + organizationId: seeded.organization.id, + apiKey: `tr_stg_${seeded.project.slug}`, + pkApiKey: `pk_stg_${seeded.project.slug}`, + shortcode: `s${seeded.project.slug.slice(0, 6)}`, + }, + }); + const third = await create({ seeded, environmentId: otherEnv.id }); + expect(third.ok).toBe(true); + } + ); + + postgresTest( + "refuses a 4th active watch in the same chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + for (const runId of ["run_1", "run_2", "run_3"]) { + const created = await create({ seeded, spec: { ...RUN_START, runId } }); + expect(created.ok).toBe(true); + } + + const fourth = await create({ seeded, spec: { ...RUN_START, runId: "run_4" } }); + expect(fourth).toMatchObject({ ok: false, code: "limit_reached" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); + } + ); + + postgresTest( + "holds the โ‰ค3 limit against four concurrent creates", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + await seedChat(seeded); + + const results = await Promise.all( + ["run_1", "run_2", "run_3", "run_4"].map((runId) => + create({ seeded, spec: { ...RUN_START, runId } }) + ) + ); + + expect(results.filter((result) => result.ok)).toHaveLength(3); + expect( + results.filter((result) => !result.ok && result.code === "limit_reached") + ).toHaveLength(1); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); + } + ); +}); + +describe("authorizeWatchEnvironment", () => { + postgresTest( + "passes for a member and fails once membership is gone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + + const params = { + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + environmentId: seeded.environment.id, + }; + + expect((await authorizeWatchEnvironment(params)).ok).toBe(true); + + await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); + expect(await authorizeWatchEnvironment(params)).toEqual({ + ok: false, + reason: "access_revoked", + }); + } + ); + + postgresTest("fails when the feature gate is revoked", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + ctx.canAccess = false; + + expect( + await authorizeWatchEnvironment({ + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + environmentId: seeded.environment.id, + }) + ).toEqual({ ok: false, reason: "access_revoked" }); + }); + + postgresTest( + "fails when the snapshot names a different project", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + const other = await seed(prisma, "other"); + + expect( + await authorizeWatchEnvironment({ + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: other.project.id, + environmentId: seeded.environment.id, + }) + ).toEqual({ ok: false, reason: "access_revoked" }); + } + ); +}); + +describe("run_failed creation", () => { + const RUN_FAILED: WatchSpec = { + kind: "run_failed", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me if it fails", + }; + + postgresTest( + "watches a running run and dedups against the finished variant separately", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "runfailed"); + await seedChat(seeded); + + const failed = await create({ + seeded, + spec: RUN_FAILED, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, + }); + expect(failed.ok).toBe(true); + if (!failed.ok || !failed.watching) return; + expect(failed.identity).toBe("run_failed:run_1"); + + const finished = await create({ + seeded, + spec: { ...RUN_FAILED, kind: "run_finished" } as WatchSpec, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, + }); + expect(finished.ok).toBe(true); + if (!finished.ok || !finished.watching) return; + expect(finished.identity).toBe("run_finished:run_1"); + } + ); + + postgresTest( + "answers outright, with no watch row, once the run has succeeded", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "runfailed-done"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: RUN_FAILED, + checkDeps: { + readRun: async () => + runRow({ status: "COMPLETED_SUCCESSFULLY", completedAt: new Date() }), + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + if (result.watching) return; + expect(result.immediate.result).toBe("terminal_unsatisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); +}); + +describe("the queue pack creation", () => { + const QUEUE = "task/my-task"; + + const BELOW: WatchSpec = { + kind: "queue_depth_below", + queue: QUEUE, + threshold: 100, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when it's back below 100", + }; + + const STALLED: WatchSpec = { + kind: "queue_stalled", + queue: QUEUE, + ticks: 3, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it stops moving", + }; + + const AGE: WatchSpec = { + kind: "queue_oldest_age", + queue: QUEUE, + thresholdMinutes: 5, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if runs wait longer than 5 minutes", + }; + + postgresTest( + "creates each kind with its own identity on the same queue", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuepack"); + await seedChat(seeded); + + const busy = { + readQueueDepth: async () => ({ + depth: 780, + source: "live_queue" as const, + current: true, + }), + }; + + const below = await create({ seeded, spec: BELOW, checkDeps: busy }); + expect(below.ok && below.watching).toBe(true); + if (!below.ok || !below.watching) return; + expect(below.identity).toBe(`queue_depth_below:${QUEUE}:100`); + + const stalled = await create({ seeded, spec: STALLED, checkDeps: busy }); + expect(stalled.ok && stalled.watching).toBe(true); + if (!stalled.ok || !stalled.watching) return; + expect(stalled.identity).toBe(`queue_stalled:${QUEUE}`); + + const age = await create({ seeded, spec: AGE, checkDeps: busy }); + expect(age.ok && age.watching).toBe(true); + if (!age.ok || !age.watching) return; + expect(age.identity).toBe(`queue_oldest_age:${QUEUE}:5`); + + const drain = await create({ + seeded, + spec: { ...BELOW, kind: "backlog_drain" } as WatchSpec, + checkDeps: busy, + }); + expect(drain.ok).toBe(false); + if (drain.ok) return; + expect(drain.code).toBe("limit_reached"); + } + ); + + postgresTest( + "dedups the same SLA and allows a different one", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queueage"); + await seedChat(seeded); + + const first = await create({ seeded, spec: AGE }); + expect(first.ok && first.watching).toBe(true); + + const same = await create({ seeded, spec: AGE }); + expect(same.ok).toBe(false); + if (same.ok) return; + expect(same.code).toBe("duplicate"); + + const other = await create({ seeded, spec: { ...AGE, thresholdMinutes: 30 } as WatchSpec }); + expect(other.ok && other.watching).toBe(true); + if (!other.ok || !other.watching) return; + expect(other.identity).toBe(`queue_oldest_age:${QUEUE}:30`); + } + ); + + postgresTest( + "answers a back-below ask outright when the queue is already quiet", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuebelow"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: BELOW, + checkDeps: { + readQueueDepth: async () => ({ depth: 4, source: "live_queue", current: true }), + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + if (result.watching) return; + expect(result.immediate.result).toBe("satisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); + + postgresTest( + "round-trips the stall state through the row's existing facts column", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuestall"); + await seedChat(seeded); + + const created = await create({ + seeded, + spec: STALLED, + checkDeps: { + readQueueDepth: async () => ({ depth: 42, source: "live_queue", current: true }), + }, + }); + expect(created.ok && created.watching).toBe(true); + if (!created.ok || !created.watching) return; + + const facts = { queue: QUEUE, depth: 42, notDecreasingStreak: 2, ticks: 3 }; + await recordWatchCheck(ctx.agentDb, { + id: created.watchId, + lastResult: { + result: "pending", + facts, + observed: { + kind: "queue_stalled", + verified: true, + depth: 42, + notDecreasingStreak: 2, + ticks: 3, + }, + final: false, + }, + }); + + const row = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(previousCheckFacts(row?.lastResult)).toEqual(facts); + + await recordWatchCheck(ctx.agentDb, { + id: created.watchId, + lastResult: { checkFailed: true, detail: "clickhouse down", previous: facts }, + }); + const afterGap = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(previousCheckFacts(afterGap?.lastResult)).toEqual(facts); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.routes.test.ts b/apps/webapp/test/dashboardAgentWatches.routes.test.ts new file mode 100644 index 00000000000..74f9fd86b97 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.routes.test.ts @@ -0,0 +1,731 @@ +import { + createChat, + getWatch, + listActiveWatchesForChat, + recordWatchCheck, + type DashboardAgentDb, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import { previousCheckFacts } from "~/services/dashboardAgentWatchChecks"; +import { + DashboardAgentWatchesTestHarness, + RUN_START, + type DashboardAgentWatchesTestContext, + type Seeded, +} from "./helpers/dashboardAgentWatchesTestHelpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted( + (): DashboardAgentWatchesTestContext => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined, + triggered: [], + }) +); + +vi.mock("~/services/uatRoutePreamble.server", () => ({ + authenticateUatOrApiRequest: async () => + ctx.actor + ? { + authenticationResult: { + type: "personalAccessToken", + result: { userId: ctx.actor.userId }, + }, + userActor: ctx.actor, + } + : undefined, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => ctx.canAccess, +})); + +vi.mock("@trigger.dev/sdk", async (importOriginal) => { + const actual = await importOriginal>(); + return { + ...actual, + TriggerClient: class { + tasks = { + trigger: async (taskId: string) => { + ctx.triggered.push(taskId); + return { id: "run_test" }; + }, + }; + }, + }; +}); + +const SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.SESSION_SECRET = SESSION_SECRET; +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; +process.env.DASHBOARD_AGENT_SECRET_KEY = "test-dashboard-agent-secret"; + +const { createDashboardAgentWatch } = await import("~/services/dashboardAgentWatches.server"); +const { action: checkAction } = + await import("~/routes/api.v1.dashboard-agent.watches.$watchId.check"); +const { action: createAction } = await import("~/routes/api.v1.dashboard-agent.watches"); +const { signDashboardAgentWatchToken } = await import("~/services/dashboardAgentWatchToken.server"); +const { loader: alertsLoader, action: alertsAction } = + await import("~/routes/api.v1.dashboard-agent.alerts"); +const { action: alertChannelAction } = + await import("~/routes/api.v1.dashboard-agent.alerts.$channelId"); +const { findProjectBySlug } = await import("~/models/project.server"); +const { DASHBOARD_AGENT_WATCH_ALERT_TYPE } = + await import("~/services/dashboardAgentWatchAlerts.server"); + +const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); +const boot = harness.boot.bind(harness); +const seed = harness.seed.bind(harness); +const seedChat = harness.seedChat.bind(harness); +const create = harness.create.bind(harness); + +beforeEach(() => harness.reset()); +afterEach(() => harness.close()); + +describe("the createWatch endpoint's authorization", () => { + function post(body: unknown) { + return createAction({ + request: new Request("https://example.com/api/v1/dashboard-agent/watches", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }), + params: {}, + context: {}, + }); + } + + const validBody = (chatId: string) => ({ spec: RUN_START, chatId }); + + postgresTest("401s without a delegated token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const response = await post(validBody("chat_1")); + expect(response.status).toBe(401); + }); + + postgresTest("403s for any other client's token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "adapter"); + ctx.actor = { userId: seeded.user.id, client: "cli", environmentId: seeded.environment.id }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "forbidden_client" }); + }); + + postgresTest( + "refuses a chat the authenticated user doesn't own, writing nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const owner = await seed(prisma, "owner"); + const stranger = await seed(prisma, "stranger"); + await createChat(ctx.agentDb, { + id: "chat_victim", + organizationId: owner.organization.id, + userId: owner.user.id, + }); + + ctx.actor = { + userId: stranger.user.id, + client: "dashboard-agent", + environmentId: stranger.environment.id, + }; + + const response = await post(validBody("chat_victim")); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ code: "chat_not_found" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_victim" })).toHaveLength( + 0 + ); + } + ); + + postgresTest( + "refuses a token with no environment scope", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "noscope"); + await seedChat(seeded, "chat_1"); + ctx.actor = { userId: seeded.user.id, client: "dashboard-agent" }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses a body naming a different environment than the token's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "mismatch"); + const other = await seed(prisma, "othermismatch"); + await seedChat(seeded, "chat_1"); + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const response = await post({ + ...validBody("chat_1"), + environmentId: other.environment.id, + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "binds to the token's environment, not the chat's stored context", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "binding"); + const otherProject = await prisma.project.create({ + data: { + name: `${seeded.project.slug}_b`, + slug: `${seeded.project.slug}_b`, + organizationId: seeded.organization.id, + externalRef: `proj_${seeded.project.slug}_b`, + }, + }); + const otherEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: otherProject.id, + organizationId: seeded.organization.id, + apiKey: `tr_prod_${otherProject.slug}`, + pkApiKey: `pk_prod_${otherProject.slug}`, + shortcode: `b${otherProject.slug.slice(0, 6)}`, + }, + }); + + await createChat(ctx.agentDb, { + id: "chat_1", + organizationId: seeded.organization.id, + userId: seeded.user.id, + metadata: { + context: { + environmentId: seeded.environment.id, + projectRef: seeded.project.externalRef, + }, + }, + }); + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: otherEnvironment.id, + }; + + const response = await post({ + ...validBody("chat_1"), + projectRef: seeded.project.externalRef, + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses an environment in another org than the chat's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "crossorg"); + const other = await seed(prisma, "otherorg"); + await prisma.orgMember.create({ + data: { + organizationId: other.organization.id, + userId: seeded.user.id, + role: "ADMIN", + }, + }); + await seedChat(seeded, "chat_1"); + + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: other.environment.id, + }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); +}); + +describe("the check endpoint", () => { + function request(token: string, body: unknown = {}) { + return new Request("https://example.com/api/v1/dashboard-agent/watches/x/check", { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + } + + async function activeWatch(seeded: Seeded, spec?: WatchSpec) { + const result = await create({ seeded, spec }); + if (!result.ok) throw new Error(`watch not created: ${result.code}`); + return result; + } + + function tokenFor(watchId: string, expiresAt: Date) { + return signDashboardAgentWatchToken(SESSION_SECRET, { watchId, expiresAt }); + } + + postgresTest("401s on a bad token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + + const response = await checkAction({ + request: request("tr_daw_nonsense"), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(401); + }); + + postgresTest("403s when the token names another watch", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor("watch_someone_else", watch.expiresAt); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "watch_mismatch" }); + }); + + postgresTest("answers a check and records what it saw", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.result).toBe("terminal_unsatisfied"); + + // Arming the chain goes through the stubbed client, never a real trigger. + expect(ctx.triggered).toContain("dashboard-agent-watch-batch"); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + expect(row?.lastCheckedAt).not.toBeNull(); + expect(row?.tickCount).toBe(0); + expect(row?.status).toBe("active"); + }); + + postgresTest( + "refuses an ordinary check after expiry but allows the final one in grace", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 minute' where id = $1`, + watch.watchId + ); + + const token = await tokenFor(watch.watchId, watch.expiresAt); + + const refused = await checkAction({ + request: request(token, {}), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(refused.status).toBe(403); + expect(await refused.json()).toMatchObject({ code: "expired" }); + + const allowed = await checkAction({ + request: request(token, { final: true }), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(allowed.status).toBe(200); + } + ); + + postgresTest( + "cancels the watch on revoked access, without reading environment data", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "access_revoked" }); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + expect(row).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + expect(row?.tickCount).toBe(0); + expect(row?.lastResult).toBeNull(); + } + ); + + postgresTest( + "a check that couldn't read anything leaves the row's last look and facts alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + + // The queue exists, so the check gets past the target read and fails on the depth + // read: there is no live queue or analytics store behind this environment. + const queue = "task/stalling"; + await prisma.taskQueue.create({ + data: { + runtimeEnvironmentId: seeded.environment.id, + projectId: seeded.project.id, + name: queue, + friendlyId: `queue_${Math.random().toString(36).slice(2, 10)}`, + orderableName: queue, + }, + }); + + const watch = await activeWatch(seeded, { + kind: "queue_stalled", + queue, + ticks: 3, + checkEveryMinutes: 5, + maxHours: 6, + note: "tell me if the queue stops moving", + }); + + // Two no-progress checks already behind it, last looked at an hour ago. + const checkedAt = new Date(Date.now() - 60 * 60 * 1000); + await recordWatchCheck(ctx.agentDb, { + id: watch.watchId, + lastCheckedAt: checkedAt, + lastResult: { + result: "pending", + facts: { queue, depth: 412, notDecreasingStreak: 2, ticks: 3 }, + }, + }); + + const token = await tokenFor(watch.watchId, watch.expiresAt); + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ result: "unavailable" }); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + // Nothing was checked, so the watch is still due at the next tick. + expect(row?.lastCheckedAt?.getTime()).toBe(checkedAt.getTime()); + // And the streak the earlier ticks built is still there to be continued. + expect(previousCheckFacts(row?.lastResult)).toMatchObject({ + depth: 412, + notDecreasingStreak: 2, + }); + }, + 120_000 + ); + + postgresTest("403s once the watch is terminal", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set status = 'cancelled' where id = $1`, + watch.watchId + ); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "cancelled" }); + }); +}); + +describe("the agent's alert boundary", () => { + /** A second, plain member of the same organization. */ + async function seedMember(prisma: PrismaClient, seeded: Seeded) { + const member = await prisma.user.create({ + data: { + email: `member_${Math.random().toString(36).slice(2, 10)}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: member.id, role: "MEMBER" }, + }); + return member; + } + + async function seedOutsider(prisma: PrismaClient) { + return prisma.user.create({ + data: { + email: `outsider_${Math.random().toString(36).slice(2, 10)}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + } + + async function seedWatchChannel(prisma: PrismaClient, seeded: Seeded, email: string) { + return prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, + name: `Watch alerts for ${email}`, + projectId: seeded.project.id, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE as never], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email }, + deduplicationKey: `dashboard-agent-watch:${email}`, + }, + }); + } + + function listRequest(chatId: string) { + return { + request: new Request( + `https://app.trigger.dev/api/v1/dashboard-agent/alerts?chatId=${chatId}`, + { headers: { Authorization: "Bearer tr_uat_test" } } + ), + params: {}, + context: {} as never, + } as never; + } + + function createRequest(body: Record) { + return { + request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/alerts", { + method: "POST", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify(body), + }), + params: {}, + context: {} as never, + } as never; + } + + function deleteRequest(channelId: string, body: Record) { + return { + request: new Request(`https://app.trigger.dev/api/v1/dashboard-agent/alerts/${channelId}`, { + method: "DELETE", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify(body), + }), + params: { channelId }, + context: {} as never, + } as never; + } + + postgresTest( + "the dashboard lets any organization member manage a project's alerts", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-policy"); + const member = await seedMember(prisma, seeded); + const outsider = await seedOutsider(prisma); + + // The whole of the Alerts page's authorization, for list, create and delete alike. + expect( + await findProjectBySlug(seeded.organization.slug, seeded.project.slug, member.id) + ).not.toBeNull(); + expect( + await findProjectBySlug(seeded.organization.slug, seeded.project.slug, outsider.id) + ).toBeNull(); + } + ); + + postgresTest( + "a plain member reads and writes watch alerts through the agent, an outsider reads nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-member"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + await seedWatchChannel(prisma, seeded, member.email); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const listed = (await alertsLoader(listRequest("chat_member"))) as Response; + expect(listed.status).toBe(200); + // The same channel the Alerts page would show this member. + expect((await listed.json()).alerts).toHaveLength(1); + + // An outsider has no chat here and no membership, so nothing resolves. + ctx.actor = { + userId: (await seedOutsider(prisma)).id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const refused = (await alertsLoader(listRequest("chat_member"))) as Response; + expect(refused.status).toBe(404); + } + ); + + postgresTest( + "the agent only ever subscribes the caller's own address", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-create"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const own = (await alertsAction( + createRequest({ chatId: "chat_member", channel: "email" }) + )) as Response; + expect(own.status).toBe(200); + expect((await own.json()).target).toBe(member.email); + + // The Alerts page would let this member add anyone; the agent may not. + const other = (await alertsAction( + createRequest({ + chatId: "chat_member", + channel: "email", + email: "someone-else@example.com", + }) + )) as Response; + expect(other.status).toBe(400); + expect(await other.json()).toMatchObject({ code: "email_not_allowed" }); + + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(1); + } + ); + + postgresTest( + "the agent's delete only takes the watch type off a watch channel", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-delete"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + const watchChannel = await seedWatchChannel(prisma, seeded, member.email); + + // A channel the agent never created and has no business touching. + const runAlerts = await prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, + name: "Run failures", + projectId: seeded.project.id, + alertTypes: ["TASK_RUN"], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email: member.email }, + }, + }); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const removed = (await alertChannelAction( + deleteRequest(watchChannel.id, { chatId: "chat_member" }) + )) as Response; + expect(removed.status).toBe(200); + expect(await removed.json()).toMatchObject({ ok: true, disabledChannel: true }); + + // The Alerts page would let a member delete this outright; the agent gets a 404. + const untouched = (await alertChannelAction( + deleteRequest(runAlerts.id, { chatId: "chat_member" }) + )) as Response; + expect(untouched.status).toBe(404); + expect( + await prisma.projectAlertChannel.findFirst({ where: { id: runAlerts.id } }) + ).toMatchObject({ enabled: true, alertTypes: ["TASK_RUN"] }); + + // An outsider can't reach the channel at all. + ctx.actor = { + userId: (await seedOutsider(prisma)).id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const refused = (await alertChannelAction( + deleteRequest(watchChannel.id, { chatId: "chat_member" }) + )) as Response; + expect(refused.status).toBe(404); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.submit.test.ts b/apps/webapp/test/dashboardAgentWatches.submit.test.ts new file mode 100644 index 00000000000..cc0109efe57 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.submit.test.ts @@ -0,0 +1,617 @@ +import { + appendChatMessageOnce, + countUserMessages, + getWatch, + getWatchSubmission, + listActiveWatchesForChat, + recordWatchSubmissionOutcome, + transitionWatchCondition, + type DashboardAgentDb, +} from "@internal/dashboard-agent-db"; +import type { WatchDraft } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; +import { + DashboardAgentWatchesTestHarness, + RUN_START, + draftFor, + type DashboardAgentWatchesTestContext, + type Seeded, +} from "./helpers/dashboardAgentWatchesTestHelpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted( + (): DashboardAgentWatchesTestContext => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined, + triggered: [], + }) +); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; + +const { createDashboardAgentWatch, submitDashboardAgentWatch } = + await import("~/services/dashboardAgentWatches.server"); +const { subscribeUserToWatchAlerts } = await import("~/services/dashboardAgentWatchAlerts.server"); + +const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); +const boot = harness.boot.bind(harness); +const seed = harness.seed.bind(harness); +const authenticated = harness.authenticated.bind(harness); +const seedChat = harness.seedChat.bind(harness); +const runRow = harness.runRow.bind(harness); +const fakeCheckDeps = harness.fakeCheckDeps.bind(harness); +const create = harness.create.bind(harness); +const storedMessages = harness.storedMessages.bind(harness); + +beforeEach(() => harness.reset()); +afterEach(() => harness.close()); + +function submit(args: { + seeded: Seeded; + draft?: WatchDraft; + chatId?: string; + clientRequestId?: string; + checkDeps?: Partial; + subscribed?: boolean; + /** Replaces the fake outright, so a test can hand the submit the real subscribe. */ + subscribe?: typeof subscribeUserToWatchAlerts; + onSchedule?: () => void; + /** Wraps the creation step, so a test can die at the exact point after it. */ + create?: typeof createDashboardAgentWatch; +}) { + return submitDashboardAgentWatch({ + environment: authenticated(args.seeded), + userId: args.seeded.user.id, + organizationId: args.seeded.organization.id, + chatId: args.chatId, + clientRequestId: args.clientRequestId ?? "wreq_1", + draft: args.draft ?? draftFor(RUN_START), + deps: { + configured: () => true, + checkDeps: () => fakeCheckDeps(args.checkDeps), + scheduleTick: async () => args.onSchedule?.(), + ...(args.create ? { create: args.create } : {}), + subscribe: + args.subscribe ?? + (async () => + args.subscribed === false + ? { ok: false, reason: "dashboard_agent_disabled" } + : { ok: true, email: args.seeded.user.email }), + }, + }); +} + +describe("the watch card submit", () => { + postgresTest( + "records what the user confirmed before the watch, and confirms it after", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit"); + await seedChat(seeded); + + const result = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor(RUN_START, { investigateOnAttention: true }), + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(true); + expect(result.repaired).toBe(false); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${result.watchId}`, + ]); + // The consent record is the user's, and it states the condition and the lifetime. + expect(stored?.[0]).toMatchObject({ role: "user" }); + expect(JSON.stringify(stored?.[0])).toContain("Watch run run_1 until it starts."); + expect(JSON.stringify(stored?.[0])).toContain("Investigate straight away"); + expect(result.messages.map((message) => message.id)).toEqual( + stored?.map((message) => message.id) + ); + } + ); + + postgresTest( + "leaves a repairable state when the confirmation never lands, and the retry repairs it", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-repair"); + await seedChat(seeded); + + // The crash state: the request record is written and the watch is live, but the + // process died before the confirmation was appended. + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + message: { id: "watch-request:wreq_1", role: "user", parts: [] } as never, + }); + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok || !created.watching) return; + + const retry = await submit({ seeded, chatId: "chat_1", clientRequestId: "wreq_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(created.watchId); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${created.watchId}`, + ]); + + // Still exactly one watch: the repair loaded it rather than creating another. + const active = await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" }); + expect(active).toHaveLength(1); + } + ); + + postgresTest( + "a retried submit duplicates neither record", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-retry"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + const second = await submit({ seeded, chatId: "chat_1" }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) return; + expect(second.repaired).toBe(true); + expect(second.watchId).toBe(first.watchId); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a genuinely different request still conflicts", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-conflict"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok) return; + + // Same condition, so the same identity, but a different window: not a retry. + const longer = await submit({ + seeded, + chatId: "chat_1", + clientRequestId: "wreq_2", + draft: draftFor({ ...RUN_START, maxHours: 6 }), + }); + expect(longer).toMatchObject({ ok: false, code: "duplicate", existingId: first.watchId }); + + // Same spec, different consent: also not a retry. + const investigating = await submit({ + seeded, + chatId: "chat_1", + clientRequestId: "wreq_3", + draft: draftFor(RUN_START, { investigateOnAttention: true }), + }); + expect(investigating).toMatchObject({ ok: false, code: "duplicate" }); + + // The refused attempts are recorded under their own consent records, so the + // transcript never shows a request with no answer. + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + "watch-request:wreq_2", + "watch-confirmation:refused:wreq_2", + "watch-request:wreq_3", + "watch-confirmation:refused:wreq_3", + ]); + } + ); + + postgresTest( + "a fresh panel's retry reuses the chat the first attempt created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-fresh"); + + const first = await submit({ seeded, clientRequestId: "wreq_fresh" }); + const second = await submit({ seeded, clientRequestId: "wreq_fresh" }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) return; + expect(second.chatId).toBe(first.chatId); + + const stored = await storedMessages(seeded, first.chatId); + expect(stored).toHaveLength(2); + } + ); + + postgresTest( + "an answered condition records the request and a one-shot result, and never a watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-oneshot"); + await seedChat(seeded); + + const result = await submit({ + seeded, + chatId: "chat_1", + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + expect(result.watchId).toBeNull(); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + "watch-confirmation:one-shot:wreq_1", + ]); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + /** Every watch row for a chat, terminal ones included. `listActiveWatchesForChat` can't see those. */ + async function countWatchRows(prisma: PrismaClient, chatId: string) { + const rows = await prisma.$queryRawUnsafe>( + `select count(*)::bigint as count from trigger_dashboard_agent.watches where chat_id = $1`, + chatId + ); + return Number(rows[0]?.count ?? 0); + } + + postgresTest( + "a retry after the watch has already fired creates no second watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-fired"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok || !first.watchId) return; + + // The watch resolves and leaves the active set, so a duplicate check would find + // nothing. Only the ledger still knows this request already ran. + await transitionWatchCondition(ctx.agentDb, { + id: first.watchId, + resolution: "condition_met", + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(first.watchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a retry of an answered one-shot never becomes a watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-oneshot-retry"); + await seedChat(seeded); + + const first = await submit({ + seeded, + chatId: "chat_1", + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + expect(first.ok && first.watching === false).toBe(true); + + // The world moved on: the same condition would now be pending, so a re-evaluation + // would start a real watch. The recorded outcome is replayed instead. + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.watching).toBe(false); + expect(retry.watchId).toBeNull(); + expect(retry.repaired).toBe(true); + expect(await countWatchRows(prisma, "chat_1")).toBe(0); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + "watch-confirmation:one-shot:wreq_1", + ]); + } + ); + + postgresTest( + "the same request id carrying a different draft is a conflict", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-hash"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok) return; + + const changed = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor({ ...RUN_START, maxHours: 6 }), + }); + expect(changed).toMatchObject({ ok: false, code: "request_conflict" }); + + // A conflict writes nothing at all: no watch, and no record under the request. + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a pending submission converges on the watch its first attempt created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-converge"); + await seedChat(seeded); + + // The crash state the ledger exists for: the row is reserved, the watch is live + // under the reserved id, and the process died before the outcome was written. + let reservedWatchId = ""; + await expect( + submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + await createDashboardAgentWatch(createParams); + throw new Error("died after the watch was created"); + }, + }) + ).rejects.toThrow("died after the watch was created"); + + const pending = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(pending).toMatchObject({ state: "pending", watchId: reservedWatchId }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + // Reached the reserved row rather than creating another. + expect(retry.watchId).toBe(reservedWatchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const settled = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(settled).toMatchObject({ state: "created", watchId: reservedWatchId }); + } + ); + + postgresTest( + "converging on a watch that already fired confirms the outcome, not 'watching'", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-converge-fired"); + await seedChat(seeded); + + let reservedWatchId = ""; + await expect( + submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + await createDashboardAgentWatch(createParams); + throw new Error("died after the watch was created"); + }, + }) + ).rejects.toThrow("died after the watch was created"); + + // The watch ran and woke the chat before anyone retried the submit. + await transitionWatchCondition(ctx.agentDb, { + id: reservedWatchId, + resolution: "condition_met", + observedOutcome: { kind: "run_start", verified: true, status: "EXECUTING", started: true }, + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + // Still one row, still the same watch: adoption is not refused. + expect(retry.watchId).toBe(reservedWatchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const parts = retry.messages.at(-1)?.parts ?? []; + const block = (parts[0] as any).data.blocks[0]; + expect(block.outcome).toBe("already_true"); + expect(block.headline).not.toContain("Watching"); + expect(block.lifetime).toBeNull(); + } + ); + + postgresTest( + "a refusal that wins the race leaves no live watch behind", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-refused-race"); + await seedChat(seeded); + + // A concurrent attempt refuses this submission after the watch exists under the + // reserved id, so the ledger's winner keeps naming that id. + let reservedWatchId = ""; + const result = await submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + const created = await createDashboardAgentWatch(createParams); + const refused = await recordWatchSubmissionOutcome(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + state: "refused", + refusalCode: "internal", + refusalError: "That watch couldn't be started.", + }); + expect(refused).toMatchObject({ state: "refused", watchId: reservedWatchId }); + return created; + }, + }); + + // The user is told nothing is being watched, so nothing may be watching. + expect(result.ok).toBe(false); + const row = await getWatch(ctx.agentDb, { id: reservedWatchId }); + expect(row).toMatchObject({ status: "cancelled", cancelReason: "superseded" }); + } + ); + + postgresTest( + "the consent record never spends a message from the cap", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-quota"); + await seedChat(seeded); + + await submit({ seeded, chatId: "chat_1" }); + + expect( + await countUserMessages(ctx.agentDb, { + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).toBe(0); + } + ); + + postgresTest( + "a replay repeats the recorded email outcome and subscribes nobody", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-external-replay"); + await seedChat(seeded); + + const draft = draftFor(RUN_START, { notifyExternally: true }); + + // The first attempt asked for email and couldn't get it, so `unavailable` is what + // the transcript says and what the ledger records. + const first = await submit({ seeded, chatId: "chat_1", draft, subscribed: false }); + expect(first.ok).toBe(true); + if (!first.ok) return; + expect(JSON.stringify(first.messages)).toContain("I couldn't add email notifications"); + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ state: "created", externalNotificationStatus: "unavailable" }); + + const transcript = await storedMessages(seeded, "chat_1"); + + // The retry gets the real subscribe, which would succeed here. A replay that took the + // decision again would leave a channel row and an `enabled` answer the transcript โ€” + // append-once, so never rewritten โ€” contradicts for good. + let subscribeCalls = 0; + const retry = await submit({ + seeded, + chatId: "chat_1", + draft, + subscribe: async (subscribeParams) => { + subscribeCalls++; + return subscribeUserToWatchAlerts(subscribeParams); + }, + }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(first.watchId); + expect(subscribeCalls).toBe(0); + + expect(JSON.stringify(retry.messages)).toContain("I couldn't add email notifications"); + expect(JSON.stringify(retry.messages)).not.toContain("You'll get an email"); + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(0); + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ externalNotificationStatus: "unavailable" }); + + // The symptom: what the user is told after a refresh has to agree with the answer. + expect(await storedMessages(seeded, "chat_1")).toEqual(transcript); + } + ); + + postgresTest( + "a replay repeats the recorded 'Watching' confirmation after the watch has fired", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-replay-fired"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok || !first.watchId) return; + + await transitionWatchCondition(ctx.agentDb, { + id: first.watchId, + resolution: "condition_met", + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + + // The recorded outcome is replayed, never decided again: the append-once + // confirmation in the transcript says "Watching", so the answer has to as well. + const parts = retry.messages.at(-1)?.parts ?? []; + const block = (parts[0] as any).data.blocks[0]; + expect(block.outcome).toBe("watching"); + expect(block.headline).toContain("Watching"); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.test.ts b/apps/webapp/test/dashboardAgentWatches.test.ts deleted file mode 100644 index 7cfccf0a2f2..00000000000 --- a/apps/webapp/test/dashboardAgentWatches.test.ts +++ /dev/null @@ -1,3421 +0,0 @@ -import { - appendChatMessageOnce, - armWatchBatch, - cancelWatch, - chatExists, - claimWatchBatchTick, - claimWatchDelivery, - claimWatchTick, - getWatchSubmission, - listActiveWatchesForBatch, - listWatchBatchGroupsToArm, - stopWatchBatch, - countUnreadWatchWakes, - countUserMessages, - createChat, - createDashboardAgentDb, - getChatMessages, - getWatch, - listActiveWatchesForChat, - listChatIdsWithUnreadWakes, - listRecentWatchWakes, - markWatchDelivered, - readWatchWakeFeed, - recordWatchCheck, - recordWatchSubmissionOutcome, - releaseWatchDelivery, - transitionWatchCondition, - WATCH_DELIVERY_CLAIM_STALE_MS, - type DashboardAgentDb, - type DashboardAgentDbClient, - type Watch, -} from "@internal/dashboard-agent-db"; -import { applyDashboardAgentMigrations } from "@internal/dashboard-agent-db/testing"; -import type { WatchDraft, WatchSpec } from "@internal/dashboard-agent-contracts"; -import { postgresTest } from "@internal/testcontainers"; -import type { PrismaClient } from "@trigger.dev/database"; -import { afterEach, beforeEach, describe, expect, vi } from "vitest"; -import { - previousCheckFacts, - type WatchCheckDeps, - type WatchRunRow, -} from "~/services/dashboardAgentWatchChecks"; - -// Every test here boots a container and replays the migrations inside its own budget, -// which does not fit vitest's 5s default on a loaded CI host. -vi.setConfig({ testTimeout: 60_000 }); - -const ctx = vi.hoisted(() => ({ - prisma: undefined as unknown as PrismaClient, - agentDb: undefined as unknown as DashboardAgentDb, - canAccess: true, - actor: undefined as undefined | { userId: string; client?: string; environmentId?: string }, - /** Every task id the suite would have triggered for real. */ - triggered: [] as string[], -})); - -vi.mock("~/services/uatRoutePreamble.server", () => ({ - authenticateUatOrApiRequest: async () => - ctx.actor - ? { - authenticationResult: { - type: "personalAccessToken", - result: { userId: ctx.actor.userId }, - }, - userActor: ctx.actor, - } - : undefined, -})); - -vi.mock("~/db.server", () => { - const proxy = new Proxy( - {}, - { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } - ); - return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; -}); - -vi.mock("~/services/dashboardAgentDb.server", () => ({ - get dashboardAgentDb() { - return ctx.agentDb; - }, -})); - -vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ - canAccessDashboardAgent: async () => ctx.canAccess, -})); - -// The routes drive the real service, which builds a TriggerClient from .env โ€” so an unmocked -// suite triggers actual runs against whatever origin .env names. -vi.mock("@trigger.dev/sdk", async (importOriginal) => { - const actual = await importOriginal>(); - return { - ...actual, - TriggerClient: class { - tasks = { - trigger: async (taskId: string) => { - ctx.triggered.push(taskId); - return { id: "run_test" }; - }, - }; - }, - }; -}); - -const SESSION_SECRET = "test-session-secret-for-watch-tokens"; -process.env.SESSION_SECRET = SESSION_SECRET; -// The agent's subscribe endpoint refuses without an email transport configured. -process.env.ALERT_FROM_EMAIL = "alerts@example.com"; -process.env.ALERT_EMAIL_TRANSPORT = "smtp"; -// Arming a batch chain builds a (stubbed) client only when this is set; unset in CI, it would -// no-op and the check test's trigger assertion would never see the batch task. -process.env.DASHBOARD_AGENT_SECRET_KEY = "test-dashboard-agent-secret"; - -const { - armDashboardAgentWatchBatch, - authorizeWatchEnvironment, - cancelDashboardAgentWatch, - createDashboardAgentWatch, - deleteChatWithWatches, - listActiveWatchesForChats, - submitDashboardAgentWatch, - watchBatchStaleMs, -} = await import("~/services/dashboardAgentWatches.server"); -const { action: checkAction } = - await import("~/routes/api.v1.dashboard-agent.watches.$watchId.check"); -const { action: createAction } = await import("~/routes/api.v1.dashboard-agent.watches"); -const { action: batchCheckAction } = - await import("~/routes/api.v1.dashboard-agent.watches.batch-check"); -const { - rearmDashboardAgentWatchBatches, - sweepDashboardAgentWatches, - WATCH_DELIVERY_GRACE_MS, - WATCH_EXPIRY_GRACE_MS, -} = await import("~/services/dashboardAgentWatchSweep.server"); -const { runWatchBatchCheck } = await import("~/services/dashboardAgentWatchBatch.server"); -const { signDashboardAgentWatchBatchToken, signDashboardAgentWatchToken } = - await import("~/services/dashboardAgentWatchToken.server"); -const { loader: alertsLoader, action: alertsAction } = - await import("~/routes/api.v1.dashboard-agent.alerts"); -const { action: alertChannelAction } = - await import("~/routes/api.v1.dashboard-agent.alerts.$channelId"); -const { findProjectBySlug } = await import("~/models/project.server"); -const { DASHBOARD_AGENT_WATCH_ALERT_TYPE, subscribeUserToWatchAlerts } = - await import("~/services/dashboardAgentWatchAlerts.server"); - -let agentDbClient: DashboardAgentDbClient | undefined; - -async function boot(prisma: PrismaClient, connectionUri: string) { - ctx.prisma = prisma; - await applyDashboardAgentMigrations((statement) => prisma.$executeRawUnsafe(statement)); - // A pool, not a single connection: the concurrent-create test needs the advisory lock to span connections. - agentDbClient = createDashboardAgentDb(connectionUri, { max: 8 }); - ctx.agentDb = agentDbClient.db; -} - -async function seed(prisma: PrismaClient, slugBase: string) { - const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; - const user = await prisma.user.create({ - data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, - }); - const organization = await prisma.organization.create({ data: { title: slug, slug } }); - await prisma.orgMember.create({ - data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, - }); - const project = await prisma.project.create({ - data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, - }); - const environment = await prisma.runtimeEnvironment.create({ - data: { - slug: "prod", - type: "PRODUCTION", - projectId: project.id, - organizationId: organization.id, - apiKey: `tr_prod_${slug}`, - pkApiKey: `pk_prod_${slug}`, - shortcode: `p${slug.slice(0, 6)}`, - }, - }); - return { user, organization, project, environment }; -} - -type Seeded = Awaited>; - -function authenticated(seeded: Seeded) { - return { - id: seeded.environment.id, - organizationId: seeded.organization.id, - projectId: seeded.project.id, - slug: "prod", - type: "PRODUCTION", - project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, - organization: { id: seeded.organization.id, slug: seeded.organization.slug }, - } as any; -} - -async function seedChat(seeded: Seeded, chatId = "chat_1") { - await createChat(ctx.agentDb, { - id: chatId, - organizationId: seeded.organization.id, - userId: seeded.user.id, - }); - return chatId; -} - -function runRow(overrides: Partial = {}): WatchRunRow { - return { - friendlyId: "run_1", - status: "PENDING", - queue: "task/my-task", - createdAt: new Date(), - queuedAt: null, - startedAt: null, - completedAt: null, - delayUntil: null, - ...overrides, - }; -} - -/** Injected readers. Defaults keep every condition pending with a live target. */ -function fakeCheckDeps(overrides: Partial = {}): WatchCheckDeps { - return { - readRun: async () => runRow(), - queueExists: async () => true, - readQueueDepth: async () => ({ depth: 7, source: "live_queue", current: true }), - readQueueOldestAge: async () => ({ ageMs: 30_000, source: "live_queue", current: true }), - readErrorRecurrence: async () => null, - readHealth: async () => ({ trustworthy: true, severity: "warn" }), - ...overrides, - }; -} - -const RUN_START: WatchSpec = { - kind: "run_start", - runId: "run_1", - checkEveryMinutes: 1, - maxHours: 2, - note: "tell me when it starts", -}; - -const BACKLOG: WatchSpec = { - kind: "backlog_drain", - queue: "task/my-task", - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me when it drains", -}; - -/** A run that exists for target validation and is gone when the immediate check reads it. */ -function readRunOnce(first: WatchRunRow) { - let calls = 0; - return async () => (calls++ === 0 ? first : null); -} - -function create(args: { - seeded: Seeded; - spec?: WatchSpec; - chatId?: string; - environmentId?: string; - investigateOnAttention?: boolean; - watchId?: string; - checkDeps?: Partial; - scheduled?: Array<{ watchId: string; token: string; tick: number }>; - onSchedule?: () => void; -}) { - const environment = authenticated(args.seeded); - return createDashboardAgentWatch({ - environment: args.environmentId ? { ...environment, id: args.environmentId } : environment, - userId: args.seeded.user.id, - chatId: args.chatId ?? "chat_1", - spec: args.spec ?? RUN_START, - investigateOnAttention: args.investigateOnAttention, - watchId: args.watchId, - deps: { - configured: () => true, - checkDeps: () => fakeCheckDeps(args.checkDeps), - scheduleTick: async (params) => { - args.onSchedule?.(); - args.scheduled?.push({ - watchId: params.watchId, - token: params.token, - tick: params.tick, - }); - }, - }, - }); -} - -beforeEach(() => { - ctx.canAccess = true; - ctx.actor = undefined; -}); - -afterEach(async () => { - await agentDbClient?.close(); - agentDbClient = undefined; -}); - -describe("createDashboardAgentWatch", () => { - postgresTest( - "creates an active watch and schedules its first tick", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const scheduled: Array<{ watchId: string; token: string; tick: number }> = []; - const result = await create({ seeded, scheduled }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.status).toBe("active"); - expect(result.identity).toBe("run_start:run_1"); - expect(result.immediate).toBeUndefined(); - - expect(scheduled).toHaveLength(1); - expect(scheduled[0]!.watchId).toBe(result.watchId); - expect(scheduled[0]!.tick).toBe(1); - expect(scheduled[0]!.token.startsWith("tr_daw_")).toBe(true); - - const row = await getWatch(ctx.agentDb, { id: result.watchId }); - expect(row).toMatchObject({ - status: "active", - deliveryStatus: "not_required", - environmentId: seeded.environment.id, - projectId: seeded.project.id, - organizationId: seeded.organization.id, - userId: seeded.user.id, - tickCount: 0, - investigateOnAttention: false, - projectRef: seeded.project.externalRef, - }); - } - ); - - postgresTest( - "records the investigate-on-attention consent when the caller asks for it", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const result = await create({ seeded, investigateOnAttention: true }); - - expect(result.ok).toBe(true); - if (!result.ok || !result.watching) return; - const row = await getWatch(ctx.agentDb, { id: result.watchId }); - expect(row?.investigateOnAttention).toBe(true); - expect(result.identity).toBe("run_start:run_1"); - } - ); - - postgresTest( - "stamps a server-set `since` on an error_recurrence watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const before = Date.now(); - const result = await create({ - seeded, - spec: { - kind: "error_recurrence", - fingerprint: "fp_1", - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me if it comes back", - }, - }); - expect(result.ok).toBe(true); - if (!result.ok) return; - - const row = await getWatch(ctx.agentDb, { id: result.watchId }); - const since = (row?.spec as { since?: string } | undefined)?.since; - expect(since).toBeDefined(); - expect(new Date(since!).getTime()).toBeGreaterThanOrEqual(before - 1000); - } - ); - - postgresTest( - "answers with a one-shot result and writes no row when the condition already holds", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - let ticks = 0; - const result = await create({ - seeded, - checkDeps: { - readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), - }, - onSchedule: () => { - ticks += 1; - }, - }); - - expect(result.ok).toBe(true); - if (!result.ok || result.watching) throw new Error("expected a one-shot result"); - expect(result.immediate.result).toBe("satisfied"); - expect(result.immediate.observed).toMatchObject({ kind: "run_start", started: true }); - expect(ticks).toBe(0); - - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - expect( - await listActiveWatchesForChats({ - chatIds: ["chat_1"], - organizationId: seeded.organization.id, - userId: seeded.user.id, - }) - ).toEqual({}); - } - ); - - postgresTest( - "answers with a one-shot result when the condition can no longer happen", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const result = await create({ - seeded, - checkDeps: { readRun: readRunOnce(runRow({ status: "QUEUED" })) }, - }); - - expect(result.ok).toBe(true); - if (!result.ok || result.watching) throw new Error("expected a one-shot result"); - expect(result.immediate.result).toBe("terminal_unsatisfied"); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "refuses a duplicate before running the immediate check", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const first = await create({ seeded }); - expect(first.ok).toBe(true); - - let checks = 0; - const second = await create({ - seeded, - checkDeps: { - readRun: async () => { - checks += 1; - return runRow({ status: "EXECUTING", startedAt: new Date() }); - }, - }, - }); - - expect(second).toMatchObject({ ok: false, code: "duplicate" }); - expect(checks).toBe(1); - } - ); - - postgresTest( - "cancels the row silently when the first tick can't be scheduled", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const result = await create({ - seeded, - onSchedule: () => { - throw new Error("no agent project"); - }, - }); - - expect(result).toMatchObject({ ok: false, code: "internal" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - const rows = await ctx.prisma.$queryRawUnsafe< - { status: string; cancel_reason: string; delivery_status: string }[] - >( - `select status, cancel_reason, delivery_status - from trigger_dashboard_agent.watches where chat_id = 'chat_1'` - ); - expect(rows).toMatchObject([ - { - status: "cancelled", - cancel_reason: "scheduling_failed", - delivery_status: "not_required", - }, - ]); - } - ); - - postgresTest( - "rejects a target that doesn't exist, writing nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const result = await create({ - seeded, - spec: BACKLOG, - checkDeps: { queueExists: async () => false }, - }); - - expect(result).toMatchObject({ ok: false, code: "invalid_target" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "dedups the same condition and allows it in another environment", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const first = await create({ seeded }); - expect(first.ok).toBe(true); - - const second = await create({ seeded }); - expect(second).toMatchObject({ ok: false, code: "duplicate" }); - if (!second.ok && first.ok) expect(second.existingId).toBe(first.watchId); - - const otherEnv = await prisma.runtimeEnvironment.create({ - data: { - slug: "stg", - type: "STAGING", - projectId: seeded.project.id, - organizationId: seeded.organization.id, - apiKey: `tr_stg_${seeded.project.slug}`, - pkApiKey: `pk_stg_${seeded.project.slug}`, - shortcode: `s${seeded.project.slug.slice(0, 6)}`, - }, - }); - const third = await create({ seeded, environmentId: otherEnv.id }); - expect(third.ok).toBe(true); - } - ); - - postgresTest( - "refuses a 4th active watch in the same chat", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - for (const runId of ["run_1", "run_2", "run_3"]) { - const created = await create({ seeded, spec: { ...RUN_START, runId } }); - expect(created.ok).toBe(true); - } - - const fourth = await create({ seeded, spec: { ...RUN_START, runId: "run_4" } }); - expect(fourth).toMatchObject({ ok: false, code: "limit_reached" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); - } - ); - - postgresTest( - "holds the โ‰ค3 limit against four concurrent creates", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "race"); - await seedChat(seeded); - - const results = await Promise.all( - ["run_1", "run_2", "run_3", "run_4"].map((runId) => - create({ seeded, spec: { ...RUN_START, runId } }) - ) - ); - - expect(results.filter((result) => result.ok)).toHaveLength(3); - expect( - results.filter((result) => !result.ok && result.code === "limit_reached") - ).toHaveLength(1); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); - } - ); -}); - -describe("the createWatch endpoint's authorization", () => { - function post(body: unknown) { - return createAction({ - request: new Request("https://example.com/api/v1/dashboard-agent/watches", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(body), - }), - params: {}, - context: {}, - }); - } - - const validBody = (chatId: string) => ({ spec: RUN_START, chatId }); - - postgresTest("401s without a delegated token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const response = await post(validBody("chat_1")); - expect(response.status).toBe(401); - }); - - postgresTest("403s for any other client's token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "adapter"); - ctx.actor = { userId: seeded.user.id, client: "cli", environmentId: seeded.environment.id }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "forbidden_client" }); - }); - - postgresTest( - "refuses a chat the authenticated user doesn't own, writing nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const owner = await seed(prisma, "owner"); - const stranger = await seed(prisma, "stranger"); - await createChat(ctx.agentDb, { - id: "chat_victim", - organizationId: owner.organization.id, - userId: owner.user.id, - }); - - ctx.actor = { - userId: stranger.user.id, - client: "dashboard-agent", - environmentId: stranger.environment.id, - }; - - const response = await post(validBody("chat_victim")); - expect(response.status).toBe(404); - expect(await response.json()).toMatchObject({ code: "chat_not_found" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_victim" })).toHaveLength( - 0 - ); - } - ); - - postgresTest( - "refuses a token with no environment scope", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "noscope"); - await seedChat(seeded, "chat_1"); - ctx.actor = { userId: seeded.user.id, client: "dashboard-agent" }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "invalid_target" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "refuses a body naming a different environment than the token's", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "mismatch"); - const other = await seed(prisma, "othermismatch"); - await seedChat(seeded, "chat_1"); - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const response = await post({ - ...validBody("chat_1"), - environmentId: other.environment.id, - }); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "binds to the token's environment, not the chat's stored context", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "binding"); - const otherProject = await prisma.project.create({ - data: { - name: `${seeded.project.slug}_b`, - slug: `${seeded.project.slug}_b`, - organizationId: seeded.organization.id, - externalRef: `proj_${seeded.project.slug}_b`, - }, - }); - const otherEnvironment = await prisma.runtimeEnvironment.create({ - data: { - slug: "prod", - type: "PRODUCTION", - projectId: otherProject.id, - organizationId: seeded.organization.id, - apiKey: `tr_prod_${otherProject.slug}`, - pkApiKey: `pk_prod_${otherProject.slug}`, - shortcode: `b${otherProject.slug.slice(0, 6)}`, - }, - }); - - await createChat(ctx.agentDb, { - id: "chat_1", - organizationId: seeded.organization.id, - userId: seeded.user.id, - metadata: { - context: { - environmentId: seeded.environment.id, - projectRef: seeded.project.externalRef, - }, - }, - }); - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: otherEnvironment.id, - }; - - const response = await post({ - ...validBody("chat_1"), - projectRef: seeded.project.externalRef, - }); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "refuses an environment in another org than the chat's", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "crossorg"); - const other = await seed(prisma, "otherorg"); - await prisma.orgMember.create({ - data: { - organizationId: other.organization.id, - userId: seeded.user.id, - role: "ADMIN", - }, - }); - await seedChat(seeded, "chat_1"); - - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: other.environment.id, - }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(404); - expect(await response.json()).toMatchObject({ code: "invalid_target" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); -}); - -describe("the chat cascade and the list view", () => { - postgresTest( - "deleting a chat soft-deletes it and cancels its active watches in one call", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "cascade"); - await seedChat(seeded, "chat_1"); - await seedChat(seeded, "chat_2"); - - const mine = await create({ seeded, chatId: "chat_1" }); - const theirs = await create({ seeded, chatId: "chat_2" }); - expect(mine.ok && theirs.ok).toBe(true); - if (!mine.ok || !theirs.ok) return; - - expect( - await deleteChatWithWatches({ - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }) - ).toEqual({ - deleted: true, - cancelledWatches: 1, - }); - - expect( - await chatExists(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }) - ).toBe(false); - expect(await getWatch(ctx.agentDb, { id: mine.watchId })).toMatchObject({ - status: "cancelled", - cancelReason: "chat_deleted", - deliveryStatus: "not_required", - }); - expect(await getWatch(ctx.agentDb, { id: theirs.watchId })).toMatchObject({ - status: "active", - }); - } - ); - - postgresTest( - "a user's own cancel leaves one neutral line in the chat, and only one", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "usercancel"); - await seedChat(seeded, "chat_1"); - - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - if (!created.ok) return; - - const cancel = () => - cancelDashboardAgentWatch({ - watchId: created.watchId, - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - - expect(await cancel()).toMatchObject({ - cancelled: true, - messages: [ - { - id: `watch-cancelled:${created.watchId}`, - role: "assistant", - parts: [{ type: "text", text: "Stopped watching run run_1." }], - }, - ], - }); - expect(await getWatch(ctx.agentDb, { id: created.watchId })).toMatchObject({ - status: "cancelled", - cancelReason: "user", - deliveryStatus: "not_required", - }); - expect(await storedMessages(seeded, "chat_1")).toMatchObject([ - { id: `watch-cancelled:${created.watchId}`, role: "assistant" }, - ]); - - // The row is no longer active, so the second cancel writes nothing at all. - expect(await cancel()).toEqual({ cancelled: false, messages: [] }); - expect(await storedMessages(seeded, "chat_1")).toHaveLength(1); - } - ); - - postgresTest( - "a chat delete cancels its watches without a line in the chat", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "silentcancel"); - await seedChat(seeded, "chat_1"); - - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - - await deleteChatWithWatches({ - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - - const rows = await ctx.prisma.$queryRawUnsafe<{ message_id: string }[]>( - `select message_id from trigger_dashboard_agent.chat_messages where chat_id = 'chat_1'` - ); - expect(rows).toEqual([]); - } - ); - - postgresTest( - "aggregates active watches per chat in one query", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "chips"); - await seedChat(seeded, "chat_1"); - await seedChat(seeded, "chat_2"); - - const a = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_1" } }); - const b = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_2" } }); - const c = await create({ seeded, chatId: "chat_2" }); - expect(a.ok && b.ok && c.ok).toBe(true); - - const byChat = await listActiveWatchesForChats({ - chatIds: ["chat_1", "chat_2", "chat_missing"], - organizationId: seeded.organization.id, - userId: seeded.user.id, - }); - expect(byChat.chat_1).toHaveLength(2); - expect(byChat.chat_2).toHaveLength(1); - expect(byChat.chat_missing).toBeUndefined(); - expect(byChat.chat_2![0]).toMatchObject({ - identity: "run_start:run_1", - status: "active", - kind: "run_start", - note: RUN_START.note, - }); - - if (a.ok) await cancelWatch(ctx.agentDb, { id: a.watchId, reason: "user" }); - if (b.ok) await cancelWatch(ctx.agentDb, { id: b.watchId, reason: "user" }); - expect( - ( - await listActiveWatchesForChats({ - chatIds: ["chat_1"], - organizationId: seeded.organization.id, - userId: seeded.user.id, - }) - ).chat_1 - ).toBeUndefined(); - } - ); - - postgresTest("returns nothing for an empty chat list", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - expect( - await listActiveWatchesForChats({ chatIds: [], organizationId: "org_x", userId: "user_x" }) - ).toEqual({}); - }); -}); - -describe("unread watch wakes", () => { - postgresTest( - "only signals a wake once its delivery landed", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "unread"); - await seedChat(seeded, "chat_1"); - - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - if (!created.ok) return; - - const scope = { organizationId: seeded.organization.id, userId: seeded.user.id }; - const recent = { ...scope, deliveredAfter: new Date(Date.now() - 15 * 60 * 1000) }; - - if (!created.watching) throw new Error("expected a watch"); - await transitionWatchCondition(ctx.agentDb, { - id: created.watchId, - resolution: "condition_met", - }); - expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(0); - expect(await listRecentWatchWakes(ctx.agentDb, recent)).toEqual([]); - expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set()); - - await markWatchDelivered(ctx.agentDb, { id: created.watchId }); - expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(1); - expect(await listRecentWatchWakes(ctx.agentDb, recent)).toMatchObject([ - { watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }, - ]); - expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set(["chat_1"])); - - // The poll's single query answers both halves the same way. - expect(await readWatchWakeFeed(ctx.agentDb, recent)).toMatchObject({ - unreadWakes: 1, - wakes: [{ watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }], - }); - - // An unread wake from before the window still counts, but isn't narrated again. - expect( - await readWatchWakeFeed(ctx.agentDb, { - ...scope, - deliveredAfter: new Date(Date.now() + 60_000), - }) - ).toMatchObject({ unreadWakes: 1, wakes: [] }); - } - ); -}); - -describe("authorizeWatchEnvironment", () => { - postgresTest( - "passes for a member and fails once membership is gone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "auth"); - - const params = { - userId: seeded.user.id, - organizationId: seeded.organization.id, - projectId: seeded.project.id, - environmentId: seeded.environment.id, - }; - - expect((await authorizeWatchEnvironment(params)).ok).toBe(true); - - await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); - expect(await authorizeWatchEnvironment(params)).toEqual({ - ok: false, - reason: "access_revoked", - }); - } - ); - - postgresTest("fails when the feature gate is revoked", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "auth"); - ctx.canAccess = false; - - expect( - await authorizeWatchEnvironment({ - userId: seeded.user.id, - organizationId: seeded.organization.id, - projectId: seeded.project.id, - environmentId: seeded.environment.id, - }) - ).toEqual({ ok: false, reason: "access_revoked" }); - }); - - postgresTest( - "fails when the snapshot names a different project", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "auth"); - const other = await seed(prisma, "other"); - - expect( - await authorizeWatchEnvironment({ - userId: seeded.user.id, - organizationId: seeded.organization.id, - projectId: other.project.id, - environmentId: seeded.environment.id, - }) - ).toEqual({ ok: false, reason: "access_revoked" }); - } - ); -}); - -describe("the watch sweep", () => { - async function overdueWatch(seeded: Seeded, chatId = "chat_1") { - const created = await create({ seeded, chatId }); - if (!created.ok) throw new Error("the watch wasn't created"); - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 hour' where id = $1`, - created.watchId - ); - return created.watchId; - } - - function sweepDeps(args: { - seeded: Seeded; - checkDeps?: Partial; - revoked?: boolean; - now?: Date; - failDelivery?: boolean; - delivered: string[]; - }) { - return { - now: () => args.now ?? new Date(), - checkDeps: () => fakeCheckDeps(args.checkDeps), - authorize: async () => - args.revoked - ? ({ ok: false, reason: "access_revoked" } as const) - : ({ ok: true, environment: authenticated(args.seeded) } as const), - deliver: async (watch: Watch) => { - if (args.failDelivery) throw new Error("the delivery couldn't be scheduled"); - args.delivered.push(watch.id); - }, - configured: () => true, - }; - } - - postgresTest( - "runs the final check on an overdue watch and fires it at the buzzer", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - const result = await sweepDashboardAgentWatches( - sweepDeps({ - seeded, - delivered, - checkDeps: { - readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), - }, - }) - ); - - expect(result).toMatchObject({ overdue: 1, fired: 1, expired: 0, cancelled: 0, failed: 0 }); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - status: "fired", - deliveryStatus: "pending", - }); - expect(delivered).toEqual([watchId]); - } - ); - - postgresTest( - "expires an overdue watch the check says hasn't happened, as verified", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); - - expect(result).toMatchObject({ overdue: 1, expired: 1, failed: 0 }); - const row = await getWatch(ctx.agentDb, { id: watchId }); - expect(row).toMatchObject({ status: "expired", deliveryStatus: "pending" }); - expect(row?.lastResult).toMatchObject({ verified: true, reason: "not_met_by_expiry" }); - expect(delivered).toEqual([watchId]); - } - ); - - postgresTest( - "cancels an overdue watch whose user lost access, and never wakes the chat", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - const result = await sweepDashboardAgentWatches( - sweepDeps({ seeded, delivered, revoked: true }) - ); - - expect(result).toMatchObject({ overdue: 1, cancelled: 1, expired: 0, fired: 0, failed: 0 }); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - status: "cancelled", - cancelReason: "access_revoked", - deliveryStatus: "not_required", - }); - expect(delivered).toEqual([]); - } - ); - - postgresTest( - "leaves a watch that is still inside its deadline alone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const created = await create({ seeded }); - expect(created.ok).toBe(true); - - const delivered: string[] = []; - const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); - - expect(result).toMatchObject({ overdue: 0, undelivered: 0 }); - expect(delivered).toEqual([]); - } - ); - - postgresTest( - "recovers a wake the delivery lost, through the real query, exactly once", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - await expect( - sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, failDelivery: true })) - ).rejects.toThrow(/failed on 1 watches/); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - status: "expired", - deliveryStatus: "pending", - }); - expect(delivered).toEqual([]); - - const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); - const second = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); - expect(second).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); - expect(delivered).toEqual([watchId]); - - await markWatchDelivered(ctx.agentDb, { id: watchId }); - const third = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); - expect(third).toMatchObject({ undelivered: 0, redelivered: 0 }); - expect(delivered).toEqual([watchId]); - } - ); - - postgresTest( - "a deliverer that died mid-delivery is recovered, but a fresh claim is left alone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] })); - - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches - set delivery_status = 'delivering', - delivery_claimed_at = now(), - last_checked_at = now() - interval '1 hour' - where id = $1`, - watchId - ); - expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] }))).toMatchObject({ - undelivered: 0, - }); - - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches - set delivery_claimed_at = now() - interval '1 hour' where id = $1`, - watchId - ); - const recovered: string[] = []; - expect( - await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: recovered })) - ).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); - expect(recovered).toEqual([watchId]); - } - ); - - postgresTest( - "leaves nothing owed for a request the immediate check already answered", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - - const created = await create({ - seeded, - checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, - }); - expect(created.ok).toBe(true); - if (!created.ok || created.watching) throw new Error("expected a one-shot result"); - - const delivered: string[] = []; - const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); - const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); - - expect(result).toMatchObject({ overdue: 0, undelivered: 0, redelivered: 0 }); - expect(delivered).toEqual([]); - } - ); - - postgresTest( - "finalizes overdue watches even with no agent to deliver to, and delivers once it's back", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - const unconfigured = await sweepDashboardAgentWatches({ - ...sweepDeps({ seeded, delivered }), - configured: () => false, - }); - - expect(unconfigured).toMatchObject({ - overdue: 1, - expired: 1, - deliveryDeferred: 1, - undelivered: 0, - redelivered: 0, - failed: 0, - }); - expect(delivered).toEqual([]); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - status: "expired", - deliveryStatus: "pending", - }); - - const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); - const restored = await sweepDashboardAgentWatches( - sweepDeps({ seeded, delivered, now: later }) - ); - expect(restored).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); - expect(delivered).toEqual([watchId]); - } - ); - - postgresTest( - "the expiry grace keeps the sweep off a watch the tick chain is still finishing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const created = await create({ seeded }); - expect(created.ok).toBe(true); - if (!created.ok) return; - - // A second past the deadline, so the chain's own final check owns this window. - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 second' where id = $1`, - created.watchId - ); - const delivered: string[] = []; - expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered }))).toMatchObject({ - overdue: 0, - }); - - const later = new Date(Date.now() + WATCH_EXPIRY_GRACE_MS + 60_000); - expect( - await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })) - ).toMatchObject({ overdue: 1, expired: 1 }); - } - ); -}); - -describe("the tick claim", () => { - postgresTest( - "claiming a generation is not an observation: only a recorded check stamps one", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "claim"); - await seedChat(seeded); - const created = await create({ seeded }); - expect(created.ok).toBe(true); - if (!created.ok) return; - - const claimed = await claimWatchTick(ctx.agentDb, { id: created.watchId, generation: 1 }); - expect(claimed).toMatchObject({ tickCount: 1, lastCheckedAt: null, lastResult: null }); - - await recordWatchCheck(ctx.agentDb, { id: created.watchId, lastResult: { pending: 4 } }); - const row = await getWatch(ctx.agentDb, { id: created.watchId }); - expect(row?.lastCheckedAt).toBeInstanceOf(Date); - expect(row?.lastResult).toMatchObject({ pending: 4 }); - expect(row?.tickCount).toBe(1); - } - ); -}); - -// The delivery claim's fencing token: a hung deliverer is taken over, so an unfenced release or mark would touch the new owner's claim. -describe("the delivery claim", () => { - async function firedWatch(seeded: Seeded) { - const created = await create({ seeded }); - expect(created.ok).toBe(true); - if (!created.ok) throw new Error("the watch wasn't created"); - const transitioned = await transitionWatchCondition(ctx.agentDb, { - id: created.watchId, - status: "fired", - lastResult: { result: "satisfied", facts: { verified: true } }, - }); - expect(transitioned).toMatchObject({ deliveryStatus: "pending" }); - return created.watchId; - } - - function staleBefore() { - return new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS); - } - - async function ageClaim(watchId: string) { - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches - set delivery_claimed_at = now() - interval '1 hour' where id = $1`, - watchId - ); - } - - postgresTest( - "a stale takeover makes the old owner's release a no-op, and the new owner delivers once", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "claim-fence"); - await seedChat(seeded); - const watchId = await firedWatch(seeded); - - const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); - expect(a).not.toBeNull(); - if (!a) return; - - await ageClaim(watchId); - const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); - expect(b).not.toBeNull(); - if (!b) return; - expect(b.claimId).not.toBe(a.claimId); - - expect( - await releaseWatchDelivery(ctx.agentDb, { id: watchId, claimId: a.claimId }) - ).toBeNull(); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - deliveryStatus: "delivering", - deliveryClaimId: b.claimId, - }); - - expect( - await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) - ).toBeNull(); - - expect( - await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) - ).toMatchObject({ deliveryStatus: "delivered" }); - expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId })).toBeNull(); - } - ); - - postgresTest( - "a late delivered-mark from the old owner completes nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "claim-late"); - await seedChat(seeded); - const watchId = await firedWatch(seeded); - - const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); - expect(a).not.toBeNull(); - if (!a) return; - await ageClaim(watchId); - const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); - expect(b).not.toBeNull(); - if (!b) return; - - expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: a.claimId })).toBeNull(); - expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - deliveryStatus: "delivering", - deliveredAt: null, - }); - - expect( - await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) - ).toMatchObject({ deliveryStatus: "delivered" }); - } - ); - - postgresTest( - "the inline path marks a pending delivery without a claim", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "claim-inline"); - await seedChat(seeded); - const watchId = await firedWatch(seeded); - - expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toMatchObject({ - deliveryStatus: "delivered", - }); - expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); - expect( - await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) - ).toBeNull(); - } - ); -}); - -describe("deleting a chat while a watch is being created", () => { - postgresTest("holds in both orders", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "race"); - - for (const deleteFirst of [true, false]) { - const chatId = `chat_${deleteFirst ? "del" : "add"}`; - await seedChat(seeded, chatId); - - const creating = () => create({ seeded, chatId }); - const deleting = () => - deleteChatWithWatches({ - chatId, - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - const [a, b] = deleteFirst - ? await Promise.all([deleting(), creating()]) - : await Promise.all([creating(), deleting()]); - expect(a).toBeDefined(); - expect(b).toBeDefined(); - - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId })).toEqual([]); - expect( - await chatExists(ctx.agentDb, { - chatId, - userId: seeded.user.id, - organizationId: seeded.organization.id, - }) - ).toBe(false); - } - }); - - postgresTest( - "refuses a create against an already-deleted chat", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "race"); - await seedChat(seeded); - await deleteChatWithWatches({ - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - - expect(await create({ seeded })).toMatchObject({ ok: false, code: "chat_not_found" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); - } - ); -}); - -describe("the check endpoint", () => { - function request(token: string, body: unknown = {}) { - return new Request("https://example.com/api/v1/dashboard-agent/watches/x/check", { - method: "POST", - headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, - body: JSON.stringify(body), - }); - } - - async function activeWatch(seeded: Seeded, spec?: WatchSpec) { - const result = await create({ seeded, spec }); - if (!result.ok) throw new Error(`watch not created: ${result.code}`); - return result; - } - - function tokenFor(watchId: string, expiresAt: Date) { - return signDashboardAgentWatchToken(SESSION_SECRET, { watchId, expiresAt }); - } - - postgresTest("401s on a bad token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - - const response = await checkAction({ - request: request("tr_daw_nonsense"), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(401); - }); - - postgresTest("403s when the token names another watch", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor("watch_someone_else", watch.expiresAt); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "watch_mismatch" }); - }); - - postgresTest("answers a check and records what it saw", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(200); - const body = await response.json(); - expect(body.result).toBe("terminal_unsatisfied"); - - // Arming the chain goes through the stubbed client, never a real trigger. - expect(ctx.triggered).toContain("dashboard-agent-watch-batch"); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - expect(row?.lastCheckedAt).not.toBeNull(); - expect(row?.tickCount).toBe(0); - expect(row?.status).toBe("active"); - }); - - postgresTest( - "refuses an ordinary check after expiry but allows the final one in grace", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - - await prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 minute' where id = $1`, - watch.watchId - ); - - const token = await tokenFor(watch.watchId, watch.expiresAt); - - const refused = await checkAction({ - request: request(token, {}), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(refused.status).toBe(403); - expect(await refused.json()).toMatchObject({ code: "expired" }); - - const allowed = await checkAction({ - request: request(token, { final: true }), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(allowed.status).toBe(200); - } - ); - - postgresTest( - "cancels the watch on revoked access, without reading environment data", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "access_revoked" }); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - expect(row).toMatchObject({ - status: "cancelled", - cancelReason: "access_revoked", - deliveryStatus: "not_required", - }); - expect(row?.tickCount).toBe(0); - expect(row?.lastResult).toBeNull(); - } - ); - - postgresTest( - "a check that couldn't read anything leaves the row's last look and facts alone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - - // The queue exists, so the check gets past the target read and fails on the depth - // read: there is no live queue or analytics store behind this environment. - const queue = "task/stalling"; - await prisma.taskQueue.create({ - data: { - runtimeEnvironmentId: seeded.environment.id, - projectId: seeded.project.id, - name: queue, - friendlyId: `queue_${Math.random().toString(36).slice(2, 10)}`, - orderableName: queue, - }, - }); - - const watch = await activeWatch(seeded, { - kind: "queue_stalled", - queue, - ticks: 3, - checkEveryMinutes: 5, - maxHours: 6, - note: "tell me if the queue stops moving", - }); - - // Two no-progress checks already behind it, last looked at an hour ago. - const checkedAt = new Date(Date.now() - 60 * 60 * 1000); - await recordWatchCheck(ctx.agentDb, { - id: watch.watchId, - lastCheckedAt: checkedAt, - lastResult: { - result: "pending", - facts: { queue, depth: 412, notDecreasingStreak: 2, ticks: 3 }, - }, - }); - - const token = await tokenFor(watch.watchId, watch.expiresAt); - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ result: "unavailable" }); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - // Nothing was checked, so the watch is still due at the next tick. - expect(row?.lastCheckedAt?.getTime()).toBe(checkedAt.getTime()); - // And the streak the earlier ticks built is still there to be continued. - expect(previousCheckFacts(row?.lastResult)).toMatchObject({ - depth: 412, - notDecreasingStreak: 2, - }); - }, - 120_000 - ); - - postgresTest("403s once the watch is terminal", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - await prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set status = 'cancelled' where id = $1`, - watch.watchId - ); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "cancelled" }); - }); -}); - -/** A configured card, with both follow-ups off unless a test turns one on. */ -function draftFor(spec: WatchSpec, followUp: Partial = {}): WatchDraft { - return { - spec, - followUp: { investigateOnAttention: false, notifyExternally: false, ...followUp }, - }; -} - -function submit(args: { - seeded: Seeded; - draft?: WatchDraft; - chatId?: string; - clientRequestId?: string; - checkDeps?: Partial; - subscribed?: boolean; - /** Replaces the fake outright, so a test can hand the submit the real subscribe. */ - subscribe?: typeof subscribeUserToWatchAlerts; - onSchedule?: () => void; - /** Wraps the creation step, so a test can die at the exact point after it. */ - create?: typeof createDashboardAgentWatch; -}) { - return submitDashboardAgentWatch({ - environment: authenticated(args.seeded), - userId: args.seeded.user.id, - organizationId: args.seeded.organization.id, - chatId: args.chatId, - clientRequestId: args.clientRequestId ?? "wreq_1", - draft: args.draft ?? draftFor(RUN_START), - deps: { - configured: () => true, - checkDeps: () => fakeCheckDeps(args.checkDeps), - scheduleTick: async () => args.onSchedule?.(), - ...(args.create ? { create: args.create } : {}), - subscribe: - args.subscribe ?? - (async () => - args.subscribed === false - ? { ok: false, reason: "dashboard_agent_disabled" } - : { ok: true, email: args.seeded.user.email }), - }, - }); -} - -function storedMessages(seeded: Seeded, chatId: string) { - return getChatMessages(ctx.agentDb, { - chatId, - userId: seeded.user.id, - organizationId: seeded.organization.id, - }) as Promise | null>; -} - -/** - * The Alerts page authorizes with `findProjectBySlug` alone (see - * `_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx`): - * every organization member may list, create and delete a project's alert channels, with - * no role check. These tests pin that policy and prove the agent's routes never write - * wider than it. - */ -describe("the agent's alert boundary", () => { - /** A second, plain member of the same organization. */ - async function seedMember(prisma: PrismaClient, seeded: Seeded) { - const member = await prisma.user.create({ - data: { - email: `member_${Math.random().toString(36).slice(2, 10)}@example.com`, - authenticationMethod: "MAGIC_LINK", - }, - }); - await prisma.orgMember.create({ - data: { organizationId: seeded.organization.id, userId: member.id, role: "MEMBER" }, - }); - return member; - } - - async function seedOutsider(prisma: PrismaClient) { - return prisma.user.create({ - data: { - email: `outsider_${Math.random().toString(36).slice(2, 10)}@example.com`, - authenticationMethod: "MAGIC_LINK", - }, - }); - } - - async function seedWatchChannel(prisma: PrismaClient, seeded: Seeded, email: string) { - return prisma.projectAlertChannel.create({ - data: { - friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, - name: `Watch alerts for ${email}`, - projectId: seeded.project.id, - alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE as never], - environmentTypes: ["PRODUCTION"], - type: "EMAIL", - properties: { email }, - deduplicationKey: `dashboard-agent-watch:${email}`, - }, - }); - } - - function listRequest(chatId: string) { - return { - request: new Request( - `https://app.trigger.dev/api/v1/dashboard-agent/alerts?chatId=${chatId}`, - { headers: { Authorization: "Bearer tr_uat_test" } } - ), - params: {}, - context: {} as never, - } as never; - } - - function createRequest(body: Record) { - return { - request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/alerts", { - method: "POST", - headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, - body: JSON.stringify(body), - }), - params: {}, - context: {} as never, - } as never; - } - - function deleteRequest(channelId: string, body: Record) { - return { - request: new Request(`https://app.trigger.dev/api/v1/dashboard-agent/alerts/${channelId}`, { - method: "DELETE", - headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, - body: JSON.stringify(body), - }), - params: { channelId }, - context: {} as never, - } as never; - } - - postgresTest( - "the dashboard lets any organization member manage a project's alerts", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-policy"); - const member = await seedMember(prisma, seeded); - const outsider = await seedOutsider(prisma); - - // The whole of the Alerts page's authorization, for list, create and delete alike. - expect( - await findProjectBySlug(seeded.organization.slug, seeded.project.slug, member.id) - ).not.toBeNull(); - expect( - await findProjectBySlug(seeded.organization.slug, seeded.project.slug, outsider.id) - ).toBeNull(); - } - ); - - postgresTest( - "a plain member reads and writes watch alerts through the agent, an outsider reads nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-member"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - await seedWatchChannel(prisma, seeded, member.email); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const listed = (await alertsLoader(listRequest("chat_member"))) as Response; - expect(listed.status).toBe(200); - // The same channel the Alerts page would show this member. - expect((await listed.json()).alerts).toHaveLength(1); - - // An outsider has no chat here and no membership, so nothing resolves. - ctx.actor = { - userId: (await seedOutsider(prisma)).id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const refused = (await alertsLoader(listRequest("chat_member"))) as Response; - expect(refused.status).toBe(404); - } - ); - - postgresTest( - "the agent only ever subscribes the caller's own address", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-create"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const own = (await alertsAction( - createRequest({ chatId: "chat_member", channel: "email" }) - )) as Response; - expect(own.status).toBe(200); - expect((await own.json()).target).toBe(member.email); - - // The Alerts page would let this member add anyone; the agent may not. - const other = (await alertsAction( - createRequest({ - chatId: "chat_member", - channel: "email", - email: "someone-else@example.com", - }) - )) as Response; - expect(other.status).toBe(400); - expect(await other.json()).toMatchObject({ code: "email_not_allowed" }); - - expect( - await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) - ).toBe(1); - } - ); - - postgresTest( - "the agent's delete only takes the watch type off a watch channel", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-delete"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - const watchChannel = await seedWatchChannel(prisma, seeded, member.email); - - // A channel the agent never created and has no business touching. - const runAlerts = await prisma.projectAlertChannel.create({ - data: { - friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, - name: "Run failures", - projectId: seeded.project.id, - alertTypes: ["TASK_RUN"], - environmentTypes: ["PRODUCTION"], - type: "EMAIL", - properties: { email: member.email }, - }, - }); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const removed = (await alertChannelAction( - deleteRequest(watchChannel.id, { chatId: "chat_member" }) - )) as Response; - expect(removed.status).toBe(200); - expect(await removed.json()).toMatchObject({ ok: true, disabledChannel: true }); - - // The Alerts page would let a member delete this outright; the agent gets a 404. - const untouched = (await alertChannelAction( - deleteRequest(runAlerts.id, { chatId: "chat_member" }) - )) as Response; - expect(untouched.status).toBe(404); - expect( - await prisma.projectAlertChannel.findFirst({ where: { id: runAlerts.id } }) - ).toMatchObject({ enabled: true, alertTypes: ["TASK_RUN"] }); - - // An outsider can't reach the channel at all. - ctx.actor = { - userId: (await seedOutsider(prisma)).id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const refused = (await alertChannelAction( - deleteRequest(watchChannel.id, { chatId: "chat_member" }) - )) as Response; - expect(refused.status).toBe(404); - } - ); -}); - -describe("the watch card submit", () => { - postgresTest( - "records what the user confirmed before the watch, and confirms it after", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit"); - await seedChat(seeded); - - const result = await submit({ - seeded, - chatId: "chat_1", - draft: draftFor(RUN_START, { investigateOnAttention: true }), - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(true); - expect(result.repaired).toBe(false); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${result.watchId}`, - ]); - // The consent record is the user's, and it states the condition and the lifetime. - expect(stored?.[0]).toMatchObject({ role: "user" }); - expect(JSON.stringify(stored?.[0])).toContain("Watch run run_1 until it starts."); - expect(JSON.stringify(stored?.[0])).toContain("Investigate straight away"); - expect(result.messages.map((message) => message.id)).toEqual( - stored?.map((message) => message.id) - ); - } - ); - - postgresTest( - "leaves a repairable state when the confirmation never lands, and the retry repairs it", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-repair"); - await seedChat(seeded); - - // The crash state: the request record is written and the watch is live, but the - // process died before the confirmation was appended. - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - message: { id: "watch-request:wreq_1", role: "user", parts: [] } as never, - }); - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - if (!created.ok || !created.watching) return; - - const retry = await submit({ seeded, chatId: "chat_1", clientRequestId: "wreq_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(created.watchId); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${created.watchId}`, - ]); - - // Still exactly one watch: the repair loaded it rather than creating another. - const active = await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" }); - expect(active).toHaveLength(1); - } - ); - - postgresTest( - "a retried submit duplicates neither record", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-retry"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - const second = await submit({ seeded, chatId: "chat_1" }); - - expect(first.ok && second.ok).toBe(true); - if (!first.ok || !second.ok) return; - expect(second.repaired).toBe(true); - expect(second.watchId).toBe(first.watchId); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a genuinely different request still conflicts", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-conflict"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok) return; - - // Same condition, so the same identity, but a different window: not a retry. - const longer = await submit({ - seeded, - chatId: "chat_1", - clientRequestId: "wreq_2", - draft: draftFor({ ...RUN_START, maxHours: 6 }), - }); - expect(longer).toMatchObject({ ok: false, code: "duplicate", existingId: first.watchId }); - - // Same spec, different consent: also not a retry. - const investigating = await submit({ - seeded, - chatId: "chat_1", - clientRequestId: "wreq_3", - draft: draftFor(RUN_START, { investigateOnAttention: true }), - }); - expect(investigating).toMatchObject({ ok: false, code: "duplicate" }); - - // The refused attempts are recorded under their own consent records, so the - // transcript never shows a request with no answer. - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - "watch-request:wreq_2", - "watch-confirmation:refused:wreq_2", - "watch-request:wreq_3", - "watch-confirmation:refused:wreq_3", - ]); - } - ); - - postgresTest( - "a fresh panel's retry reuses the chat the first attempt created", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-fresh"); - - const first = await submit({ seeded, clientRequestId: "wreq_fresh" }); - const second = await submit({ seeded, clientRequestId: "wreq_fresh" }); - - expect(first.ok && second.ok).toBe(true); - if (!first.ok || !second.ok) return; - expect(second.chatId).toBe(first.chatId); - - const stored = await storedMessages(seeded, first.chatId); - expect(stored).toHaveLength(2); - } - ); - - postgresTest( - "an answered condition records the request and a one-shot result, and never a watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-oneshot"); - await seedChat(seeded); - - const result = await submit({ - seeded, - chatId: "chat_1", - checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(false); - expect(result.watchId).toBeNull(); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - "watch-confirmation:one-shot:wreq_1", - ]); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - /** Every watch row for a chat, terminal ones included. `listActiveWatchesForChat` can't see those. */ - async function countWatchRows(prisma: PrismaClient, chatId: string) { - const rows = await prisma.$queryRawUnsafe>( - `select count(*)::bigint as count from trigger_dashboard_agent.watches where chat_id = $1`, - chatId - ); - return Number(rows[0]?.count ?? 0); - } - - postgresTest( - "a retry after the watch has already fired creates no second watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-fired"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok || !first.watchId) return; - - // The watch resolves and leaves the active set, so a duplicate check would find - // nothing. Only the ledger still knows this request already ran. - await transitionWatchCondition(ctx.agentDb, { - id: first.watchId, - resolution: "condition_met", - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(first.watchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a retry of an answered one-shot never becomes a watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-oneshot-retry"); - await seedChat(seeded); - - const first = await submit({ - seeded, - chatId: "chat_1", - checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, - }); - expect(first.ok && first.watching === false).toBe(true); - - // The world moved on: the same condition would now be pending, so a re-evaluation - // would start a real watch. The recorded outcome is replayed instead. - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.watching).toBe(false); - expect(retry.watchId).toBeNull(); - expect(retry.repaired).toBe(true); - expect(await countWatchRows(prisma, "chat_1")).toBe(0); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - "watch-confirmation:one-shot:wreq_1", - ]); - } - ); - - postgresTest( - "the same request id carrying a different draft is a conflict", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-hash"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok) return; - - const changed = await submit({ - seeded, - chatId: "chat_1", - draft: draftFor({ ...RUN_START, maxHours: 6 }), - }); - expect(changed).toMatchObject({ ok: false, code: "request_conflict" }); - - // A conflict writes nothing at all: no watch, and no record under the request. - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a pending submission converges on the watch its first attempt created", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-converge"); - await seedChat(seeded); - - // The crash state the ledger exists for: the row is reserved, the watch is live - // under the reserved id, and the process died before the outcome was written. - let reservedWatchId = ""; - await expect( - submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - await createDashboardAgentWatch(createParams); - throw new Error("died after the watch was created"); - }, - }) - ).rejects.toThrow("died after the watch was created"); - - const pending = await getWatchSubmission(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - }); - expect(pending).toMatchObject({ state: "pending", watchId: reservedWatchId }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - // Reached the reserved row rather than creating another. - expect(retry.watchId).toBe(reservedWatchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const settled = await getWatchSubmission(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - }); - expect(settled).toMatchObject({ state: "created", watchId: reservedWatchId }); - } - ); - - postgresTest( - "converging on a watch that already fired confirms the outcome, not 'watching'", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-converge-fired"); - await seedChat(seeded); - - let reservedWatchId = ""; - await expect( - submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - await createDashboardAgentWatch(createParams); - throw new Error("died after the watch was created"); - }, - }) - ).rejects.toThrow("died after the watch was created"); - - // The watch ran and woke the chat before anyone retried the submit. - await transitionWatchCondition(ctx.agentDb, { - id: reservedWatchId, - resolution: "condition_met", - observedOutcome: { kind: "run_start", verified: true, status: "EXECUTING", started: true }, - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - // Still one row, still the same watch: adoption is not refused. - expect(retry.watchId).toBe(reservedWatchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const parts = retry.messages.at(-1)?.parts ?? []; - const block = (parts[0] as any).data.blocks[0]; - expect(block.outcome).toBe("already_true"); - expect(block.headline).not.toContain("Watching"); - expect(block.lifetime).toBeNull(); - } - ); - - postgresTest( - "a refusal that wins the race leaves no live watch behind", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-refused-race"); - await seedChat(seeded); - - // A concurrent attempt refuses this submission after the watch exists under the - // reserved id, so the ledger's winner keeps naming that id. - let reservedWatchId = ""; - const result = await submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - const created = await createDashboardAgentWatch(createParams); - const refused = await recordWatchSubmissionOutcome(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - state: "refused", - refusalCode: "internal", - refusalError: "That watch couldn't be started.", - }); - expect(refused).toMatchObject({ state: "refused", watchId: reservedWatchId }); - return created; - }, - }); - - // The user is told nothing is being watched, so nothing may be watching. - expect(result.ok).toBe(false); - const row = await getWatch(ctx.agentDb, { id: reservedWatchId }); - expect(row).toMatchObject({ status: "cancelled", cancelReason: "superseded" }); - } - ); - - postgresTest( - "the consent record never spends a message from the cap", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-quota"); - await seedChat(seeded); - - await submit({ seeded, chatId: "chat_1" }); - - expect( - await countUserMessages(ctx.agentDb, { - organizationId: seeded.organization.id, - userId: seeded.user.id, - }) - ).toBe(0); - } - ); - - postgresTest( - "a replay repeats the recorded email outcome and subscribes nobody", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-external-replay"); - await seedChat(seeded); - - const draft = draftFor(RUN_START, { notifyExternally: true }); - - // The first attempt asked for email and couldn't get it, so `unavailable` is what - // the transcript says and what the ledger records. - const first = await submit({ seeded, chatId: "chat_1", draft, subscribed: false }); - expect(first.ok).toBe(true); - if (!first.ok) return; - expect(JSON.stringify(first.messages)).toContain("I couldn't add email notifications"); - expect( - await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) - ).toMatchObject({ state: "created", externalNotificationStatus: "unavailable" }); - - const transcript = await storedMessages(seeded, "chat_1"); - - // The retry gets the real subscribe, which would succeed here. A replay that took the - // decision again would leave a channel row and an `enabled` answer the transcript โ€” - // append-once, so never rewritten โ€” contradicts for good. - let subscribeCalls = 0; - const retry = await submit({ - seeded, - chatId: "chat_1", - draft, - subscribe: async (subscribeParams) => { - subscribeCalls++; - return subscribeUserToWatchAlerts(subscribeParams); - }, - }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(first.watchId); - expect(subscribeCalls).toBe(0); - - expect(JSON.stringify(retry.messages)).toContain("I couldn't add email notifications"); - expect(JSON.stringify(retry.messages)).not.toContain("You'll get an email"); - expect( - await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) - ).toBe(0); - expect( - await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) - ).toMatchObject({ externalNotificationStatus: "unavailable" }); - - // The symptom: what the user is told after a refresh has to agree with the answer. - expect(await storedMessages(seeded, "chat_1")).toEqual(transcript); - } - ); - - postgresTest( - "a replay repeats the recorded 'Watching' confirmation after the watch has fired", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-replay-fired"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok || !first.watchId) return; - - await transitionWatchCondition(ctx.agentDb, { - id: first.watchId, - resolution: "condition_met", - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - - // The recorded outcome is replayed, never decided again: the append-once - // confirmation in the transcript says "Watching", so the answer has to as well. - const parts = retry.messages.at(-1)?.parts ?? []; - const block = (parts[0] as any).data.blocks[0]; - expect(block.outcome).toBe("watching"); - expect(block.headline).toContain("Watching"); - } - ); -}); - -describe("appendChatMessageOnce", () => { - postgresTest( - "appends in order without rewriting the transcript", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "append"); - await seedChat(seeded); - - const first = { id: "watch-card:watch_1", role: "assistant", parts: [] }; - const second = { id: "watch-card:watch_2", role: "assistant", parts: [] }; - - expect( - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - message: first, - }) - ).toBe(true); - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - message: second, - }); - - const messages = await getChatMessages(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - expect(messages).toEqual([first, second]); - } - ); - - postgresTest( - "appends nothing for a chat the caller doesn't own", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "append-owner"); - await seedChat(seeded); - - expect( - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: "user_someone_else", - organizationId: seeded.organization.id, - message: { id: "watch-card:watch_1", role: "assistant", parts: [] }, - }) - ).toBe(false); - - const messages = await getChatMessages(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - expect(messages).toEqual([]); - } - ); -}); - -describe("run_failed creation", () => { - const RUN_FAILED: WatchSpec = { - kind: "run_failed", - runId: "run_1", - checkEveryMinutes: 1, - maxHours: 2, - note: "tell me if it fails", - }; - - postgresTest( - "watches a running run and dedups against the finished variant separately", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "runfailed"); - await seedChat(seeded); - - const failed = await create({ - seeded, - spec: RUN_FAILED, - checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, - }); - expect(failed.ok).toBe(true); - if (!failed.ok || !failed.watching) return; - expect(failed.identity).toBe("run_failed:run_1"); - - const finished = await create({ - seeded, - spec: { ...RUN_FAILED, kind: "run_finished" } as WatchSpec, - checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, - }); - expect(finished.ok).toBe(true); - if (!finished.ok || !finished.watching) return; - expect(finished.identity).toBe("run_finished:run_1"); - } - ); - - postgresTest( - "answers outright, with no watch row, once the run has succeeded", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "runfailed-done"); - await seedChat(seeded); - - const result = await create({ - seeded, - spec: RUN_FAILED, - checkDeps: { - readRun: async () => - runRow({ status: "COMPLETED_SUCCESSFULLY", completedAt: new Date() }), - }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(false); - if (result.watching) return; - expect(result.immediate.result).toBe("terminal_unsatisfied"); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); - } - ); -}); - -describe("the queue pack creation", () => { - const QUEUE = "task/my-task"; - - const BELOW: WatchSpec = { - kind: "queue_depth_below", - queue: QUEUE, - threshold: 100, - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me when it's back below 100", - }; - - const STALLED: WatchSpec = { - kind: "queue_stalled", - queue: QUEUE, - ticks: 3, - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me if it stops moving", - }; - - const AGE: WatchSpec = { - kind: "queue_oldest_age", - queue: QUEUE, - thresholdMinutes: 5, - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me if runs wait longer than 5 minutes", - }; - - postgresTest( - "creates each kind with its own identity on the same queue", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "queuepack"); - await seedChat(seeded); - - const busy = { - readQueueDepth: async () => ({ - depth: 780, - source: "live_queue" as const, - current: true, - }), - }; - - const below = await create({ seeded, spec: BELOW, checkDeps: busy }); - expect(below.ok && below.watching).toBe(true); - if (!below.ok || !below.watching) return; - expect(below.identity).toBe(`queue_depth_below:${QUEUE}:100`); - - const stalled = await create({ seeded, spec: STALLED, checkDeps: busy }); - expect(stalled.ok && stalled.watching).toBe(true); - if (!stalled.ok || !stalled.watching) return; - expect(stalled.identity).toBe(`queue_stalled:${QUEUE}`); - - const age = await create({ seeded, spec: AGE, checkDeps: busy }); - expect(age.ok && age.watching).toBe(true); - if (!age.ok || !age.watching) return; - expect(age.identity).toBe(`queue_oldest_age:${QUEUE}:5`); - - const drain = await create({ - seeded, - spec: { ...BELOW, kind: "backlog_drain" } as WatchSpec, - checkDeps: busy, - }); - expect(drain.ok).toBe(false); - if (drain.ok) return; - expect(drain.code).toBe("limit_reached"); - } - ); - - postgresTest( - "dedups the same SLA and allows a different one", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "queueage"); - await seedChat(seeded); - - const first = await create({ seeded, spec: AGE }); - expect(first.ok && first.watching).toBe(true); - - const same = await create({ seeded, spec: AGE }); - expect(same.ok).toBe(false); - if (same.ok) return; - expect(same.code).toBe("duplicate"); - - const other = await create({ seeded, spec: { ...AGE, thresholdMinutes: 30 } as WatchSpec }); - expect(other.ok && other.watching).toBe(true); - if (!other.ok || !other.watching) return; - expect(other.identity).toBe(`queue_oldest_age:${QUEUE}:30`); - } - ); - - postgresTest( - "answers a back-below ask outright when the queue is already quiet", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "queuebelow"); - await seedChat(seeded); - - const result = await create({ - seeded, - spec: BELOW, - checkDeps: { - readQueueDepth: async () => ({ depth: 4, source: "live_queue", current: true }), - }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(false); - if (result.watching) return; - expect(result.immediate.result).toBe("satisfied"); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); - } - ); - - postgresTest( - "round-trips the stall state through the row's existing facts column", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "queuestall"); - await seedChat(seeded); - - const created = await create({ - seeded, - spec: STALLED, - checkDeps: { - readQueueDepth: async () => ({ depth: 42, source: "live_queue", current: true }), - }, - }); - expect(created.ok && created.watching).toBe(true); - if (!created.ok || !created.watching) return; - - const facts = { queue: QUEUE, depth: 42, notDecreasingStreak: 2, ticks: 3 }; - await recordWatchCheck(ctx.agentDb, { - id: created.watchId, - lastResult: { - result: "pending", - facts, - observed: { - kind: "queue_stalled", - verified: true, - depth: 42, - notDecreasingStreak: 2, - ticks: 3, - }, - final: false, - }, - }); - - const row = await getWatch(ctx.agentDb, { id: created.watchId }); - expect(previousCheckFacts(row?.lastResult)).toEqual(facts); - - await recordWatchCheck(ctx.agentDb, { - id: created.watchId, - lastResult: { checkFailed: true, detail: "clickhouse down", previous: facts }, - }); - const afterGap = await getWatch(ctx.agentDb, { id: created.watchId }); - expect(previousCheckFacts(afterGap?.lastResult)).toEqual(facts); - } - ); -}); - -const HEALTH: WatchSpec = { - kind: "health_recovery", - report: "health", - fromSeverity: "warn", - checkEveryMinutes: 5, - maxHours: 6, - note: "tell me when health recovers", -}; - -describe("the batch chain registry", () => { - postgresTest("arms one chain per group, and only one", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batcharm"); - const now = new Date(); - - const scheduled: Array<{ epoch: number; tick: number }> = []; - const arm = () => - armDashboardAgentWatchBatch({ - environmentId: seeded.environment.id, - cadenceMinutes: 5, - now, - deps: { - schedule: async (params) => - void scheduled.push({ epoch: params.epoch, tick: params.tick }), - }, - }); - - expect(await arm()).toEqual({ running: true }); - expect(scheduled).toEqual([{ epoch: 1, tick: 1 }]); - - expect(await arm()).toEqual({ running: true }); - expect(await arm()).toEqual({ running: true }); - expect(scheduled).toHaveLength(1); - }); - - postgresTest( - "a chain whose run died is re-armed on a fresh epoch, and the zombie claims nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchdead"); - const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; - - const scheduled: Array<{ epoch: number; tick: number }> = []; - const arm = (now: Date) => - armDashboardAgentWatchBatch({ - ...group, - now, - deps: { - schedule: async (params) => - void scheduled.push({ epoch: params.epoch, tick: params.tick }), - }, - }); - - const armedAt = new Date(); - await arm(armedAt); - expect( - await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 1 }) - ).toMatchObject({ epoch: 1, generation: 1 }); - - await arm(new Date(armedAt.getTime() + 60_000)); - expect(scheduled).toHaveLength(1); - - await arm(new Date(armedAt.getTime() + watchBatchStaleMs(5) + 60_000)); - expect(scheduled).toEqual([ - { epoch: 1, tick: 1 }, - { epoch: 2, tick: 1 }, - ]); - - expect(await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 2 })).toBe( - null - ); - expect( - await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 2, generation: 1 }) - ).toMatchObject({ epoch: 2, generation: 1 }); - } - ); - - postgresTest( - "a chain that couldn't be triggered is not left marked as running", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchfail"); - const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; - - expect( - await armDashboardAgentWatchBatch({ - ...group, - deps: { - schedule: async () => { - throw new Error("the trigger failed"); - }, - }, - }) - ).toEqual({ running: false }); - - const scheduled: number[] = []; - expect( - await armDashboardAgentWatchBatch({ - ...group, - deps: { schedule: async (params) => void scheduled.push(params.epoch) }, - }) - ).toEqual({ running: true }); - expect(scheduled).toEqual([2]); - } - ); - - postgresTest( - "the re-arm backstop finds groups with active watches and no chain", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchrearm"); - await seedChat(seeded); - const created = await create({ - seeded, - spec: HEALTH, - checkDeps: { readHealth: async () => null }, - }); - expect(created.ok).toBe(true); - - const groups = await listWatchBatchGroupsToArm(ctx.agentDb); - expect(groups).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); - - const armed: Array<{ environmentId: string; cadenceMinutes: number }> = []; - expect( - await rearmDashboardAgentWatchBatches({ - configured: () => true, - arm: async (params) => { - armed.push({ - environmentId: params.environmentId, - cadenceMinutes: params.cadenceMinutes, - }); - return { running: true }; - }, - }) - ).toEqual({ stale: 1, armed: 1, failed: 0 }); - expect(armed).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); - - // The staleness window is the group's own cadence: a five-minute group goes stale 17 minutes later. - await armWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - staleBefore: new Date(), - }); - expect(await listWatchBatchGroupsToArm(ctx.agentDb)).toEqual([]); - expect( - await listWatchBatchGroupsToArm(ctx.agentDb, { - now: new Date(Date.now() + watchBatchStaleMs(5) + 60_000), - }) - ).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); - } - ); - - postgresTest( - "groups are per environment and per cadence, never mixed", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchgroup"); - await seedChat(seeded, "chat_1"); - await seedChat(seeded, "chat_2"); - expect((await create({ seeded, chatId: "chat_1", spec: HEALTH })).ok).toBe(true); - expect((await create({ seeded, chatId: "chat_2", spec: RUN_START })).ok).toBe(true); - - const five = await listActiveWatchesForBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - }); - const one = await listActiveWatchesForBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 1, - }); - - expect(five.map((watch) => watch.chatId)).toEqual(["chat_1"]); - expect(one.map((watch) => watch.chatId)).toEqual(["chat_2"]); - expect( - (await listWatchBatchGroupsToArm(ctx.agentDb)).sort( - (a, b) => a.cadenceMinutes - b.cadenceMinutes - ) - ).toEqual([ - { environmentId: seeded.environment.id, cadenceMinutes: 1 }, - { environmentId: seeded.environment.id, cadenceMinutes: 5 }, - ]); - } - ); -}); - -describe("the batch check", () => { - async function healthGroup(seeded: Seeded, count = 3) { - const ids: string[] = []; - for (let index = 0; index < count; index++) { - const chatId = `chat_${index + 1}`; - await seedChat(seeded, chatId); - const created = await create({ - seeded, - chatId, - spec: HEALTH, - // `warn` keeps them all pending, so the group stays whole for the assertions below. - checkDeps: { readHealth: async () => ({ trustworthy: true, severity: "warn" }) }, - }); - if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); - ids.push(created.watchId); - } - return ids; - } - - async function otherUsersWatch(seeded: Seeded, prisma: PrismaClient) { - const user = await prisma.user.create({ - data: { - email: `other_${seeded.organization.slug}@example.com`, - authenticationMethod: "MAGIC_LINK", - }, - }); - await prisma.orgMember.create({ - data: { organizationId: seeded.organization.id, userId: user.id, role: "MEMBER" }, - }); - await createChat(ctx.agentDb, { - id: "chat_other", - organizationId: seeded.organization.id, - userId: user.id, - }); - const created = await createDashboardAgentWatch({ - environment: authenticated(seeded), - userId: user.id, - chatId: "chat_other", - spec: HEALTH, - deps: { - configured: () => true, - checkDeps: () => - fakeCheckDeps({ readHealth: async () => ({ trustworthy: true, severity: "warn" }) }), - scheduleTick: async () => {}, - }, - }); - if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); - return { userId: user.id, watchId: created.watchId }; - } - - async function armChain(seeded: Seeded, cadenceMinutes = 5) { - const row = await armWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes, - staleBefore: new Date(), - }); - if (!row) throw new Error("the chain wasn't armed"); - return row; - } - - postgresTest( - "authorizes once and loads the shared report once for the whole group", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchcheck"); - const ids = await healthGroup(seeded); - const chain = await armChain(seeded); - - let healthReads = 0; - let authorizations = 0; - - const response = await runWatchBatchCheck( - { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain.epoch, - tick: 1, - }, - { - authorize: async () => { - authorizations++; - return { ok: true, environment: authenticated(seeded) }; - }, - checkDeps: () => - fakeCheckDeps({ - readHealth: async () => { - healthReads++; - return { trustworthy: true, severity: "warn" }; - }, - }), - } - ); - - expect(authorizations).toBe(1); - expect(healthReads).toBe(1); - - expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual([...ids].sort()); - expect(response.watches?.every((entry) => entry.result === "pending")).toBe(true); - expect(response.watches?.every((entry) => entry.tick === 1)).toBe(true); - expect(response.watches?.every((entry) => entry.token.length > 0)).toBe(true); - expect(response.continues).toBe(true); - expect(response.stale).toBeUndefined(); - - for (const id of ids) { - expect((await getWatch(ctx.agentDb, { id }))?.lastResult).toMatchObject({ - result: "pending", - final: false, - }); - } - } - ); - - postgresTest( - "authorizes each distinct user, so sharing readers never shares access", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchusers"); - await healthGroup(seeded, 2); - const other = await otherUsersWatch(seeded, prisma); - - const chain = await armChain(seeded); - const authorized: string[] = []; - - await runWatchBatchCheck( - { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, - { - authorize: async (watch) => { - authorized.push(watch.userId); - return { ok: true, environment: authenticated(seeded) }; - }, - checkDeps: () => fakeCheckDeps(), - } - ); - - expect(authorized.sort()).toEqual([other.userId, seeded.user.id].sort()); - } - ); - - postgresTest( - "cancels a watch whose user lost access, and still answers for its neighbours", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchrevoked"); - const ids = await healthGroup(seeded, 2); - const chain = await armChain(seeded); - - const response = await runWatchBatchCheck( - { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, - { - authorize: async () => ({ ok: false, reason: "access_revoked" }), - checkDeps: () => fakeCheckDeps(), - } - ); - - expect(response.watches?.every((entry) => entry.code === "access_revoked")).toBe(true); - for (const id of ids) { - expect(await getWatch(ctx.agentDb, { id })).toMatchObject({ - status: "cancelled", - cancelReason: "access_revoked", - deliveryStatus: "not_required", - }); - } - } - ); - - postgresTest( - "checks what is due, skips what isn't, and never skips a window boundary", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchdue"); - const [fresh, overdue, boundary] = await healthGroup(seeded, 3); - const chain = await armChain(seeded); - const now = new Date(); - - await recordWatchCheck(ctx.agentDb, { id: fresh!, lastCheckedAt: now }); - await recordWatchCheck(ctx.agentDb, { - id: overdue!, - lastCheckedAt: new Date(now.getTime() - 10 * 60_000), - }); - // `boundary`'s window closes before the next tick, so its final evaluation must still happen. - await recordWatchCheck(ctx.agentDb, { id: boundary!, lastCheckedAt: now }); - await prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() + interval '1 minute' where id = $1`, - boundary - ); - - const response = await runWatchBatchCheck( - { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, - { - now: () => now, - authorize: async () => ({ ok: true, environment: authenticated(seeded) }), - checkDeps: () => fakeCheckDeps(), - } - ); - - expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual( - [boundary!, overdue!].sort() - ); - expect(response.continues).toBe(true); - } - ); - - postgresTest( - "a stale tick claims nothing and checks nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchstale"); - const ids = await healthGroup(seeded, 1); - const chain = await armChain(seeded); - - const group = { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch }; - expect((await runWatchBatchCheck({ ...group, tick: 1 })).stale).toBeUndefined(); - expect((await runWatchBatchCheck({ ...group, tick: 2 })).stale).toBeUndefined(); - - const late = await runWatchBatchCheck({ ...group, tick: 1 }); - expect(late).toEqual({ stale: true }); - - expect(await runWatchBatchCheck({ ...group, epoch: chain.epoch - 1, tick: 1 })).toEqual({ - stale: true, - }); - expect((await getWatch(ctx.agentDb, { id: ids[0]! }))?.status).toBe("active"); - } - ); - - postgresTest( - "stops the chain when the group's last watch is gone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchempty"); - const ids = await healthGroup(seeded, 1); - const chain = await armChain(seeded); - await cancelWatch(ctx.agentDb, { id: ids[0]!, reason: "user" }); - - const response = await runWatchBatchCheck({ - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain.epoch, - tick: 1, - }); - - expect(response).toMatchObject({ watches: [], continues: false }); - - const rearmed = await armWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - // Deliberately in the past: only a stopped chain can be re-armed this way. - staleBefore: new Date(Date.now() - 60 * 60_000), - }); - expect(rearmed).toMatchObject({ epoch: chain.epoch + 1, status: "running" }); - } - ); - - postgresTest( - "hands the group's owed wakes back for redelivery", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchowed"); - const ids = await healthGroup(seeded, 2); - const chain = await armChain(seeded); - - await transitionWatchCondition(ctx.agentDb, { - id: ids[0]!, - resolution: "condition_met", - lastResult: { verified: true }, - }); - - const response = await runWatchBatchCheck({ - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain.epoch, - tick: 1, - }); - - const owed = response.watches?.filter((entry) => entry.deliverOnly === true) ?? []; - expect(owed.map((entry) => entry.watchId)).toEqual([ids[0]!]); - expect(owed[0]?.tick).toBe(0); - expect( - response.watches?.filter((entry) => !entry.deliverOnly).map((entry) => entry.watchId) - ).toEqual([ids[1]!]); - } - ); - - postgresTest( - "keeps the chain alive while a wake is still owed, even with nothing left to watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchowedlast"); - const ids = await healthGroup(seeded, 1); - const chain = await armChain(seeded); - const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; - - await transitionWatchCondition(ctx.agentDb, { - id: ids[0]!, - resolution: "condition_met", - lastResult: { verified: true }, - }); - - const first = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 1 }); - expect(first.continues).toBe(true); - expect(first.watches?.map((entry) => entry.deliverOnly)).toEqual([true]); - - const claim = await claimWatchDelivery(ctx.agentDb, { - id: ids[0]!, - staleBefore: new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS), - }); - await markWatchDelivered(ctx.agentDb, { id: ids[0]!, claimId: claim!.claimId }); - - const second = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 2 }); - expect(second).toMatchObject({ watches: [], continues: false }); - expect(await stopWatchBatch(ctx.agentDb, { ...group, epoch: chain.epoch })).toBe(null); - } - ); - - postgresTest( - "one watch that throws mid-evaluation costs only that watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchthrow"); - const mine = await healthGroup(seeded, 2); - const theirs = await otherUsersWatch(seeded, prisma); - const chain = await armChain(seeded); - - const response = await runWatchBatchCheck( - { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, - { - authorize: async (watch) => { - if (watch.userId === theirs.userId) throw new Error("the authorization query failed"); - return { ok: true, environment: authenticated(seeded) }; - }, - checkDeps: () => fakeCheckDeps(), - concurrency: 1, - } - ); - - const byId = new Map(response.watches?.map((entry) => [entry.watchId, entry])); - expect(byId.get(theirs.watchId)).toMatchObject({ result: "unavailable" }); - expect((await getWatch(ctx.agentDb, { id: theirs.watchId }))?.status).toBe("active"); - for (const id of mine) { - expect(byId.get(id)).toMatchObject({ result: "pending" }); - } - } - ); -}); - -describe("the batch check endpoint's authorization", () => { - function batchRequest(body: unknown, token?: string) { - return new Request("https://app.trigger.dev/api/v1/dashboard-agent/watches/batch-check", { - method: "POST", - headers: { - ...(token ? { Authorization: `Bearer ${token}` } : {}), - "Content-Type": "application/json", - }, - body: JSON.stringify(body), - }); - } - - const batchToken = (environmentId: string, cadenceMinutes: number) => - signDashboardAgentWatchBatchToken(SESSION_SECRET, { - environmentId, - cadenceMinutes, - expiresAt: new Date(Date.now() + 60 * 60_000), - }); - - postgresTest("refuses a missing or bad token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const body = { environmentId: "env_1", cadenceMinutes: 5, epoch: 1, tick: 1 }; - - expect( - (await batchCheckAction({ request: batchRequest(body), params: {}, context: {} })).status - ).toBe(401); - const watchToken = await signDashboardAgentWatchToken(SESSION_SECRET, { - watchId: "watch_1", - expiresAt: new Date(Date.now() + 60 * 60_000), - }); - expect( - (await batchCheckAction({ request: batchRequest(body, watchToken), params: {}, context: {} })) - .status - ).toBe(401); - }); - - postgresTest( - "refuses a token minted for another group", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const token = await batchToken("env_1", 5); - - const wrongCadence = await batchCheckAction({ - request: batchRequest( - { environmentId: "env_1", cadenceMinutes: 15, epoch: 1, tick: 1 }, - token - ), - params: {}, - context: {}, - }); - expect(wrongCadence.status).toBe(403); - expect(await wrongCadence.json()).toMatchObject({ code: "group_mismatch" }); - - const wrongEnvironment = await batchCheckAction({ - request: batchRequest( - { environmentId: "env_2", cadenceMinutes: 5, epoch: 1, tick: 1 }, - token - ), - params: {}, - context: {}, - }); - expect(wrongEnvironment.status).toBe(403); - } - ); - - postgresTest( - "answers a group it does own, through the real registry", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchroute"); - const chain = await armWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - staleBefore: new Date(), - }); - const token = await batchToken(seeded.environment.id, 5); - - const response = await batchCheckAction({ - request: batchRequest( - { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain!.epoch, - tick: 1, - }, - token - ), - params: {}, - context: {}, - }); - - expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ watches: [], continues: false }); - expect( - await stopWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain!.epoch, - }) - ).toBe(null); - } - ); -}); diff --git a/apps/webapp/test/helpers/dashboardAgentWatchesTestHelpers.ts b/apps/webapp/test/helpers/dashboardAgentWatchesTestHelpers.ts new file mode 100644 index 00000000000..4562860bc10 --- /dev/null +++ b/apps/webapp/test/helpers/dashboardAgentWatchesTestHelpers.ts @@ -0,0 +1,219 @@ +import { + createChat, + createDashboardAgentDb, + getChatMessages, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { applyDashboardAgentMigrations } from "@internal/dashboard-agent-db/testing"; +import type { WatchDraft, WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { WatchCheckDeps, WatchRunRow } from "~/services/dashboardAgentWatchChecks"; +import type { createDashboardAgentWatch as CreateDashboardAgentWatchFunction } from "~/services/dashboardAgentWatches.server"; + +export type DashboardAgentWatchesTestContext = { + prisma: PrismaClient; + agentDb: DashboardAgentDb; + canAccess: boolean; + actor: undefined | { userId: string; client?: string; environmentId?: string }; + /** Every task id the suite would have triggered for real. */ + triggered: string[]; +}; + +async function seedDashboardAgentWatchTestData(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 6)}`, + }, + }); + return { user, organization, project, environment }; +} + +export type Seeded = Awaited>; + +export const RUN_START: WatchSpec = { + kind: "run_start", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when it starts", +}; + +export const BACKLOG: WatchSpec = { + kind: "backlog_drain", + queue: "task/my-task", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when it drains", +}; + +export const HEALTH: WatchSpec = { + kind: "health_recovery", + report: "health", + fromSeverity: "warn", + checkEveryMinutes: 5, + maxHours: 6, + note: "tell me when health recovers", +}; + +/** A run that exists for target validation and is gone when the immediate check reads it. */ +export function readRunOnce(first: WatchRunRow) { + let calls = 0; + return async () => (calls++ === 0 ? first : null); +} + +/** A configured card, with both follow-ups off unless a test turns one on. */ +export function draftFor( + spec: WatchSpec, + followUp: Partial = {} +): WatchDraft { + return { + spec, + followUp: { investigateOnAttention: false, notifyExternally: false, ...followUp }, + }; +} + +type CreateDashboardAgentWatch = typeof CreateDashboardAgentWatchFunction; + +export class DashboardAgentWatchesTestHarness { + private agentDbClient: DashboardAgentDbClient | undefined; + + constructor( + private readonly ctx: DashboardAgentWatchesTestContext, + private readonly createDashboardAgentWatch: CreateDashboardAgentWatch + ) {} + + reset() { + this.ctx.canAccess = true; + this.ctx.actor = undefined; + this.ctx.triggered.length = 0; + } + + async boot(prisma: PrismaClient, connectionUri: string) { + this.ctx.prisma = prisma; + await applyDashboardAgentMigrations((statement) => prisma.$executeRawUnsafe(statement)); + // A pool, not a single connection: concurrent-create tests need the advisory lock to span connections. + this.agentDbClient = createDashboardAgentDb(connectionUri, { max: 8 }); + this.ctx.agentDb = this.agentDbClient.db; + } + + async close() { + await this.agentDbClient?.close(); + this.agentDbClient = undefined; + } + + seed(prisma: PrismaClient, slugBase: string) { + return seedDashboardAgentWatchTestData(prisma, slugBase); + } + + authenticated(seeded: Seeded) { + return { + id: seeded.environment.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + slug: "prod", + type: "PRODUCTION", + project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, + organization: { id: seeded.organization.id, slug: seeded.organization.slug }, + } as any; + } + + async seedChat(seeded: Seeded, chatId = "chat_1") { + await createChat(this.ctx.agentDb, { + id: chatId, + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + return chatId; + } + + runRow(overrides: Partial = {}): WatchRunRow { + return { + friendlyId: "run_1", + status: "PENDING", + queue: "task/my-task", + createdAt: new Date(), + queuedAt: null, + startedAt: null, + completedAt: null, + delayUntil: null, + ...overrides, + }; + } + + /** Injected readers. Defaults keep every condition pending with a live target. */ + fakeCheckDeps(overrides: Partial = {}): WatchCheckDeps { + return { + readRun: async () => this.runRow(), + queueExists: async () => true, + readQueueDepth: async () => ({ depth: 7, source: "live_queue", current: true }), + readQueueOldestAge: async () => ({ + ageMs: 30_000, + source: "live_queue", + current: true, + }), + readErrorRecurrence: async () => null, + readHealth: async () => ({ trustworthy: true, severity: "warn" }), + ...overrides, + }; + } + + create(args: { + seeded: Seeded; + spec?: WatchSpec; + chatId?: string; + environmentId?: string; + investigateOnAttention?: boolean; + watchId?: string; + checkDeps?: Partial; + scheduled?: Array<{ watchId: string; token: string; tick: number }>; + onSchedule?: () => void; + }) { + const environment = this.authenticated(args.seeded); + return this.createDashboardAgentWatch({ + environment: args.environmentId ? { ...environment, id: args.environmentId } : environment, + userId: args.seeded.user.id, + chatId: args.chatId ?? "chat_1", + spec: args.spec ?? RUN_START, + investigateOnAttention: args.investigateOnAttention, + watchId: args.watchId, + deps: { + configured: () => true, + checkDeps: () => this.fakeCheckDeps(args.checkDeps), + scheduleTick: async (params) => { + args.onSchedule?.(); + args.scheduled?.push({ + watchId: params.watchId, + token: params.token, + tick: params.tick, + }); + }, + }, + }); + } + + storedMessages(seeded: Seeded, chatId: string) { + return getChatMessages(this.ctx.agentDb, { + chatId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) as Promise | null>; + } +} From efb30448bcd83a48929ac9f85cfb90dfca1841e7 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 17:17:45 +0100 Subject: [PATCH 08/15] test(webapp): limit dashboard watch fixture fan-out --- .../dashboardAgentWatches.delivery.test.ts | 557 +++++++++++++ .../dashboardAgentWatches.lifecycle.test.ts | 674 +++++++++++++++- .../test/dashboardAgentWatches.routes.test.ts | 731 ------------------ .../test/dashboardAgentWatches.submit.test.ts | 617 --------------- 4 files changed, 1230 insertions(+), 1349 deletions(-) delete mode 100644 apps/webapp/test/dashboardAgentWatches.routes.test.ts delete mode 100644 apps/webapp/test/dashboardAgentWatches.submit.test.ts diff --git a/apps/webapp/test/dashboardAgentWatches.delivery.test.ts b/apps/webapp/test/dashboardAgentWatches.delivery.test.ts index 7552d1667ec..d569d39039f 100644 --- a/apps/webapp/test/dashboardAgentWatches.delivery.test.ts +++ b/apps/webapp/test/dashboardAgentWatches.delivery.test.ts @@ -5,20 +5,24 @@ import { claimWatchDelivery, claimWatchTick, countUnreadWatchWakes, + countUserMessages, getChatMessages, getWatch, + getWatchSubmission, listActiveWatchesForChat, listChatIdsWithUnreadWakes, listRecentWatchWakes, markWatchDelivered, readWatchWakeFeed, recordWatchCheck, + recordWatchSubmissionOutcome, releaseWatchDelivery, transitionWatchCondition, WATCH_DELIVERY_CLAIM_STALE_MS, type DashboardAgentDb, type Watch, } from "@internal/dashboard-agent-db"; +import type { WatchDraft } from "@internal/dashboard-agent-contracts"; import { postgresTest } from "@internal/testcontainers"; import type { PrismaClient } from "@trigger.dev/database"; import { afterEach, beforeEach, describe, expect, vi } from "vitest"; @@ -26,6 +30,7 @@ import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; import { DashboardAgentWatchesTestHarness, RUN_START, + draftFor, type DashboardAgentWatchesTestContext, type Seeded, } from "./helpers/dashboardAgentWatchesTestHelpers"; @@ -57,15 +62,19 @@ vi.mock("~/services/dashboardAgentDb.server", () => ({ })); process.env.SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; const { cancelDashboardAgentWatch, createDashboardAgentWatch, deleteChatWithWatches, listActiveWatchesForChats, + submitDashboardAgentWatch, } = await import("~/services/dashboardAgentWatches.server"); const { sweepDashboardAgentWatches, WATCH_DELIVERY_GRACE_MS, WATCH_EXPIRY_GRACE_MS } = await import("~/services/dashboardAgentWatchSweep.server"); +const { subscribeUserToWatchAlerts } = await import("~/services/dashboardAgentWatchAlerts.server"); const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); const boot = harness.boot.bind(harness); @@ -807,3 +816,551 @@ describe("appendChatMessageOnce", () => { } ); }); + +function submit(args: { + seeded: Seeded; + draft?: WatchDraft; + chatId?: string; + clientRequestId?: string; + checkDeps?: Partial; + subscribed?: boolean; + /** Replaces the fake outright, so a test can hand the submit the real subscribe. */ + subscribe?: typeof subscribeUserToWatchAlerts; + onSchedule?: () => void; + /** Wraps the creation step, so a test can die at the exact point after it. */ + create?: typeof createDashboardAgentWatch; +}) { + return submitDashboardAgentWatch({ + environment: authenticated(args.seeded), + userId: args.seeded.user.id, + organizationId: args.seeded.organization.id, + chatId: args.chatId, + clientRequestId: args.clientRequestId ?? "wreq_1", + draft: args.draft ?? draftFor(RUN_START), + deps: { + configured: () => true, + checkDeps: () => fakeCheckDeps(args.checkDeps), + scheduleTick: async () => args.onSchedule?.(), + ...(args.create ? { create: args.create } : {}), + subscribe: + args.subscribe ?? + (async () => + args.subscribed === false + ? { ok: false, reason: "dashboard_agent_disabled" } + : { ok: true, email: args.seeded.user.email }), + }, + }); +} + +describe("the watch card submit", () => { + postgresTest( + "records what the user confirmed before the watch, and confirms it after", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit"); + await seedChat(seeded); + + const result = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor(RUN_START, { investigateOnAttention: true }), + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(true); + expect(result.repaired).toBe(false); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${result.watchId}`, + ]); + // The consent record is the user's, and it states the condition and the lifetime. + expect(stored?.[0]).toMatchObject({ role: "user" }); + expect(JSON.stringify(stored?.[0])).toContain("Watch run run_1 until it starts."); + expect(JSON.stringify(stored?.[0])).toContain("Investigate straight away"); + expect(result.messages.map((message) => message.id)).toEqual( + stored?.map((message) => message.id) + ); + } + ); + + postgresTest( + "leaves a repairable state when the confirmation never lands, and the retry repairs it", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-repair"); + await seedChat(seeded); + + // The crash state: the request record is written and the watch is live, but the + // process died before the confirmation was appended. + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + message: { id: "watch-request:wreq_1", role: "user", parts: [] } as never, + }); + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok || !created.watching) return; + + const retry = await submit({ seeded, chatId: "chat_1", clientRequestId: "wreq_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(created.watchId); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${created.watchId}`, + ]); + + // Still exactly one watch: the repair loaded it rather than creating another. + const active = await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" }); + expect(active).toHaveLength(1); + } + ); + + postgresTest( + "a retried submit duplicates neither record", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-retry"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + const second = await submit({ seeded, chatId: "chat_1" }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) return; + expect(second.repaired).toBe(true); + expect(second.watchId).toBe(first.watchId); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a genuinely different request still conflicts", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-conflict"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok) return; + + // Same condition, so the same identity, but a different window: not a retry. + const longer = await submit({ + seeded, + chatId: "chat_1", + clientRequestId: "wreq_2", + draft: draftFor({ ...RUN_START, maxHours: 6 }), + }); + expect(longer).toMatchObject({ ok: false, code: "duplicate", existingId: first.watchId }); + + // Same spec, different consent: also not a retry. + const investigating = await submit({ + seeded, + chatId: "chat_1", + clientRequestId: "wreq_3", + draft: draftFor(RUN_START, { investigateOnAttention: true }), + }); + expect(investigating).toMatchObject({ ok: false, code: "duplicate" }); + + // The refused attempts are recorded under their own consent records, so the + // transcript never shows a request with no answer. + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + "watch-request:wreq_2", + "watch-confirmation:refused:wreq_2", + "watch-request:wreq_3", + "watch-confirmation:refused:wreq_3", + ]); + } + ); + + postgresTest( + "a fresh panel's retry reuses the chat the first attempt created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-fresh"); + + const first = await submit({ seeded, clientRequestId: "wreq_fresh" }); + const second = await submit({ seeded, clientRequestId: "wreq_fresh" }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) return; + expect(second.chatId).toBe(first.chatId); + + const stored = await storedMessages(seeded, first.chatId); + expect(stored).toHaveLength(2); + } + ); + + postgresTest( + "an answered condition records the request and a one-shot result, and never a watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-oneshot"); + await seedChat(seeded); + + const result = await submit({ + seeded, + chatId: "chat_1", + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + expect(result.watchId).toBeNull(); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + "watch-confirmation:one-shot:wreq_1", + ]); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + /** Every watch row for a chat, terminal ones included. `listActiveWatchesForChat` can't see those. */ + async function countWatchRows(prisma: PrismaClient, chatId: string) { + const rows = await prisma.$queryRawUnsafe>( + `select count(*)::bigint as count from trigger_dashboard_agent.watches where chat_id = $1`, + chatId + ); + return Number(rows[0]?.count ?? 0); + } + + postgresTest( + "a retry after the watch has already fired creates no second watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-fired"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok || !first.watchId) return; + + // The watch resolves and leaves the active set, so a duplicate check would find + // nothing. Only the ledger still knows this request already ran. + await transitionWatchCondition(ctx.agentDb, { + id: first.watchId, + resolution: "condition_met", + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(first.watchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a retry of an answered one-shot never becomes a watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-oneshot-retry"); + await seedChat(seeded); + + const first = await submit({ + seeded, + chatId: "chat_1", + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + expect(first.ok && first.watching === false).toBe(true); + + // The world moved on: the same condition would now be pending, so a re-evaluation + // would start a real watch. The recorded outcome is replayed instead. + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.watching).toBe(false); + expect(retry.watchId).toBeNull(); + expect(retry.repaired).toBe(true); + expect(await countWatchRows(prisma, "chat_1")).toBe(0); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + "watch-confirmation:one-shot:wreq_1", + ]); + } + ); + + postgresTest( + "the same request id carrying a different draft is a conflict", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-hash"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok) return; + + const changed = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor({ ...RUN_START, maxHours: 6 }), + }); + expect(changed).toMatchObject({ ok: false, code: "request_conflict" }); + + // A conflict writes nothing at all: no watch, and no record under the request. + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a pending submission converges on the watch its first attempt created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-converge"); + await seedChat(seeded); + + // The crash state the ledger exists for: the row is reserved, the watch is live + // under the reserved id, and the process died before the outcome was written. + let reservedWatchId = ""; + await expect( + submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + await createDashboardAgentWatch(createParams); + throw new Error("died after the watch was created"); + }, + }) + ).rejects.toThrow("died after the watch was created"); + + const pending = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(pending).toMatchObject({ state: "pending", watchId: reservedWatchId }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + // Reached the reserved row rather than creating another. + expect(retry.watchId).toBe(reservedWatchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const settled = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(settled).toMatchObject({ state: "created", watchId: reservedWatchId }); + } + ); + + postgresTest( + "converging on a watch that already fired confirms the outcome, not 'watching'", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-converge-fired"); + await seedChat(seeded); + + let reservedWatchId = ""; + await expect( + submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + await createDashboardAgentWatch(createParams); + throw new Error("died after the watch was created"); + }, + }) + ).rejects.toThrow("died after the watch was created"); + + // The watch ran and woke the chat before anyone retried the submit. + await transitionWatchCondition(ctx.agentDb, { + id: reservedWatchId, + resolution: "condition_met", + observedOutcome: { kind: "run_start", verified: true, status: "EXECUTING", started: true }, + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + // Still one row, still the same watch: adoption is not refused. + expect(retry.watchId).toBe(reservedWatchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const parts = retry.messages.at(-1)?.parts ?? []; + const block = (parts[0] as any).data.blocks[0]; + expect(block.outcome).toBe("already_true"); + expect(block.headline).not.toContain("Watching"); + expect(block.lifetime).toBeNull(); + } + ); + + postgresTest( + "a refusal that wins the race leaves no live watch behind", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-refused-race"); + await seedChat(seeded); + + // A concurrent attempt refuses this submission after the watch exists under the + // reserved id, so the ledger's winner keeps naming that id. + let reservedWatchId = ""; + const result = await submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + const created = await createDashboardAgentWatch(createParams); + const refused = await recordWatchSubmissionOutcome(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + state: "refused", + refusalCode: "internal", + refusalError: "That watch couldn't be started.", + }); + expect(refused).toMatchObject({ state: "refused", watchId: reservedWatchId }); + return created; + }, + }); + + // The user is told nothing is being watched, so nothing may be watching. + expect(result.ok).toBe(false); + const row = await getWatch(ctx.agentDb, { id: reservedWatchId }); + expect(row).toMatchObject({ status: "cancelled", cancelReason: "superseded" }); + } + ); + + postgresTest( + "the consent record never spends a message from the cap", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-quota"); + await seedChat(seeded); + + await submit({ seeded, chatId: "chat_1" }); + + expect( + await countUserMessages(ctx.agentDb, { + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).toBe(0); + } + ); + + postgresTest( + "a replay repeats the recorded email outcome and subscribes nobody", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-external-replay"); + await seedChat(seeded); + + const draft = draftFor(RUN_START, { notifyExternally: true }); + + // The first attempt asked for email and couldn't get it, so `unavailable` is what + // the transcript says and what the ledger records. + const first = await submit({ seeded, chatId: "chat_1", draft, subscribed: false }); + expect(first.ok).toBe(true); + if (!first.ok) return; + expect(JSON.stringify(first.messages)).toContain("I couldn't add email notifications"); + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ state: "created", externalNotificationStatus: "unavailable" }); + + const transcript = await storedMessages(seeded, "chat_1"); + + // The retry gets the real subscribe, which would succeed here. A replay that took the + // decision again would leave a channel row and an `enabled` answer the transcript โ€” + // append-once, so never rewritten โ€” contradicts for good. + let subscribeCalls = 0; + const retry = await submit({ + seeded, + chatId: "chat_1", + draft, + subscribe: async (subscribeParams) => { + subscribeCalls++; + return subscribeUserToWatchAlerts(subscribeParams); + }, + }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(first.watchId); + expect(subscribeCalls).toBe(0); + + expect(JSON.stringify(retry.messages)).toContain("I couldn't add email notifications"); + expect(JSON.stringify(retry.messages)).not.toContain("You'll get an email"); + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(0); + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ externalNotificationStatus: "unavailable" }); + + // The symptom: what the user is told after a refresh has to agree with the answer. + expect(await storedMessages(seeded, "chat_1")).toEqual(transcript); + } + ); + + postgresTest( + "a replay repeats the recorded 'Watching' confirmation after the watch has fired", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-replay-fired"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok || !first.watchId) return; + + await transitionWatchCondition(ctx.agentDb, { + id: first.watchId, + resolution: "condition_met", + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + + // The recorded outcome is replayed, never decided again: the append-once + // confirmation in the transcript says "Watching", so the answer has to as well. + const parts = retry.messages.at(-1)?.parts ?? []; + const block = (parts[0] as any).data.blocks[0]; + expect(block.outcome).toBe("watching"); + expect(block.headline).toContain("Watching"); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts b/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts index e54cb597406..15219ed883a 100644 --- a/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts +++ b/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts @@ -1,4 +1,5 @@ import { + createChat, getWatch, listActiveWatchesForChat, recordWatchCheck, @@ -15,6 +16,7 @@ import { RUN_START, readRunOnce, type DashboardAgentWatchesTestContext, + type Seeded, } from "./helpers/dashboardAgentWatchesTestHelpers"; vi.setConfig({ testTimeout: 60_000 }); @@ -29,6 +31,19 @@ const ctx = vi.hoisted( }) ); +vi.mock("~/services/uatRoutePreamble.server", () => ({ + authenticateUatOrApiRequest: async () => + ctx.actor + ? { + authenticationResult: { + type: "personalAccessToken", + result: { userId: ctx.actor.userId }, + }, + userActor: ctx.actor, + } + : undefined, +})); + vi.mock("~/db.server", () => { const proxy = new Proxy( {}, @@ -47,10 +62,40 @@ vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ canAccessDashboardAgent: async () => ctx.canAccess, })); -process.env.SESSION_SECRET = "test-session-secret-for-watch-tokens"; +vi.mock("@trigger.dev/sdk", async (importOriginal) => { + const actual = await importOriginal>(); + return { + ...actual, + TriggerClient: class { + tasks = { + trigger: async (taskId: string) => { + ctx.triggered.push(taskId); + return { id: "run_test" }; + }, + }; + }, + }; +}); + +const SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.SESSION_SECRET = SESSION_SECRET; +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; +process.env.DASHBOARD_AGENT_SECRET_KEY = "test-dashboard-agent-secret"; const { authorizeWatchEnvironment, createDashboardAgentWatch, listActiveWatchesForChats } = await import("~/services/dashboardAgentWatches.server"); +const { action: checkAction } = + await import("~/routes/api.v1.dashboard-agent.watches.$watchId.check"); +const { action: createAction } = await import("~/routes/api.v1.dashboard-agent.watches"); +const { signDashboardAgentWatchToken } = await import("~/services/dashboardAgentWatchToken.server"); +const { loader: alertsLoader, action: alertsAction } = + await import("~/routes/api.v1.dashboard-agent.alerts"); +const { action: alertChannelAction } = + await import("~/routes/api.v1.dashboard-agent.alerts.$channelId"); +const { findProjectBySlug } = await import("~/models/project.server"); +const { DASHBOARD_AGENT_WATCH_ALERT_TYPE } = + await import("~/services/dashboardAgentWatchAlerts.server"); const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); const boot = harness.boot.bind(harness); @@ -627,3 +672,630 @@ describe("the queue pack creation", () => { } ); }); + +describe("the createWatch endpoint's authorization", () => { + function post(body: unknown) { + return createAction({ + request: new Request("https://example.com/api/v1/dashboard-agent/watches", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }), + params: {}, + context: {}, + }); + } + + const validBody = (chatId: string) => ({ spec: RUN_START, chatId }); + + postgresTest("401s without a delegated token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const response = await post(validBody("chat_1")); + expect(response.status).toBe(401); + }); + + postgresTest("403s for any other client's token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "adapter"); + ctx.actor = { userId: seeded.user.id, client: "cli", environmentId: seeded.environment.id }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "forbidden_client" }); + }); + + postgresTest( + "refuses a chat the authenticated user doesn't own, writing nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const owner = await seed(prisma, "owner"); + const stranger = await seed(prisma, "stranger"); + await createChat(ctx.agentDb, { + id: "chat_victim", + organizationId: owner.organization.id, + userId: owner.user.id, + }); + + ctx.actor = { + userId: stranger.user.id, + client: "dashboard-agent", + environmentId: stranger.environment.id, + }; + + const response = await post(validBody("chat_victim")); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ code: "chat_not_found" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_victim" })).toHaveLength( + 0 + ); + } + ); + + postgresTest( + "refuses a token with no environment scope", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "noscope"); + await seedChat(seeded, "chat_1"); + ctx.actor = { userId: seeded.user.id, client: "dashboard-agent" }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses a body naming a different environment than the token's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "mismatch"); + const other = await seed(prisma, "othermismatch"); + await seedChat(seeded, "chat_1"); + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const response = await post({ + ...validBody("chat_1"), + environmentId: other.environment.id, + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "binds to the token's environment, not the chat's stored context", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "binding"); + const otherProject = await prisma.project.create({ + data: { + name: `${seeded.project.slug}_b`, + slug: `${seeded.project.slug}_b`, + organizationId: seeded.organization.id, + externalRef: `proj_${seeded.project.slug}_b`, + }, + }); + const otherEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: otherProject.id, + organizationId: seeded.organization.id, + apiKey: `tr_prod_${otherProject.slug}`, + pkApiKey: `pk_prod_${otherProject.slug}`, + shortcode: `b${otherProject.slug.slice(0, 6)}`, + }, + }); + + await createChat(ctx.agentDb, { + id: "chat_1", + organizationId: seeded.organization.id, + userId: seeded.user.id, + metadata: { + context: { + environmentId: seeded.environment.id, + projectRef: seeded.project.externalRef, + }, + }, + }); + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: otherEnvironment.id, + }; + + const response = await post({ + ...validBody("chat_1"), + projectRef: seeded.project.externalRef, + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses an environment in another org than the chat's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "crossorg"); + const other = await seed(prisma, "otherorg"); + await prisma.orgMember.create({ + data: { + organizationId: other.organization.id, + userId: seeded.user.id, + role: "ADMIN", + }, + }); + await seedChat(seeded, "chat_1"); + + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: other.environment.id, + }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); +}); + +describe("the check endpoint", () => { + function request(token: string, body: unknown = {}) { + return new Request("https://example.com/api/v1/dashboard-agent/watches/x/check", { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + } + + async function activeWatch(seeded: Seeded, spec?: WatchSpec) { + const result = await create({ seeded, spec }); + if (!result.ok) throw new Error(`watch not created: ${result.code}`); + return result; + } + + function tokenFor(watchId: string, expiresAt: Date) { + return signDashboardAgentWatchToken(SESSION_SECRET, { watchId, expiresAt }); + } + + postgresTest("401s on a bad token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + + const response = await checkAction({ + request: request("tr_daw_nonsense"), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(401); + }); + + postgresTest("403s when the token names another watch", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor("watch_someone_else", watch.expiresAt); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "watch_mismatch" }); + }); + + postgresTest("answers a check and records what it saw", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.result).toBe("terminal_unsatisfied"); + + // Arming the chain goes through the stubbed client, never a real trigger. + expect(ctx.triggered).toContain("dashboard-agent-watch-batch"); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + expect(row?.lastCheckedAt).not.toBeNull(); + expect(row?.tickCount).toBe(0); + expect(row?.status).toBe("active"); + }); + + postgresTest( + "refuses an ordinary check after expiry but allows the final one in grace", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 minute' where id = $1`, + watch.watchId + ); + + const token = await tokenFor(watch.watchId, watch.expiresAt); + + const refused = await checkAction({ + request: request(token, {}), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(refused.status).toBe(403); + expect(await refused.json()).toMatchObject({ code: "expired" }); + + const allowed = await checkAction({ + request: request(token, { final: true }), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(allowed.status).toBe(200); + } + ); + + postgresTest( + "cancels the watch on revoked access, without reading environment data", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "access_revoked" }); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + expect(row).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + expect(row?.tickCount).toBe(0); + expect(row?.lastResult).toBeNull(); + } + ); + + postgresTest( + "a check that couldn't read anything leaves the row's last look and facts alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + + // The queue exists, so the check gets past the target read and fails on the depth + // read: there is no live queue or analytics store behind this environment. + const queue = "task/stalling"; + await prisma.taskQueue.create({ + data: { + runtimeEnvironmentId: seeded.environment.id, + projectId: seeded.project.id, + name: queue, + friendlyId: `queue_${Math.random().toString(36).slice(2, 10)}`, + orderableName: queue, + }, + }); + + const watch = await activeWatch(seeded, { + kind: "queue_stalled", + queue, + ticks: 3, + checkEveryMinutes: 5, + maxHours: 6, + note: "tell me if the queue stops moving", + }); + + // Two no-progress checks already behind it, last looked at an hour ago. + const checkedAt = new Date(Date.now() - 60 * 60 * 1000); + await recordWatchCheck(ctx.agentDb, { + id: watch.watchId, + lastCheckedAt: checkedAt, + lastResult: { + result: "pending", + facts: { queue, depth: 412, notDecreasingStreak: 2, ticks: 3 }, + }, + }); + + const token = await tokenFor(watch.watchId, watch.expiresAt); + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ result: "unavailable" }); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + // Nothing was checked, so the watch is still due at the next tick. + expect(row?.lastCheckedAt?.getTime()).toBe(checkedAt.getTime()); + // And the streak the earlier ticks built is still there to be continued. + expect(previousCheckFacts(row?.lastResult)).toMatchObject({ + depth: 412, + notDecreasingStreak: 2, + }); + }, + 120_000 + ); + + postgresTest("403s once the watch is terminal", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set status = 'cancelled' where id = $1`, + watch.watchId + ); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "cancelled" }); + }); +}); + +describe("the agent's alert boundary", () => { + /** A second, plain member of the same organization. */ + async function seedMember(prisma: PrismaClient, seeded: Seeded) { + const member = await prisma.user.create({ + data: { + email: `member_${Math.random().toString(36).slice(2, 10)}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: member.id, role: "MEMBER" }, + }); + return member; + } + + async function seedOutsider(prisma: PrismaClient) { + return prisma.user.create({ + data: { + email: `outsider_${Math.random().toString(36).slice(2, 10)}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + } + + async function seedWatchChannel(prisma: PrismaClient, seeded: Seeded, email: string) { + return prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, + name: `Watch alerts for ${email}`, + projectId: seeded.project.id, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE as never], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email }, + deduplicationKey: `dashboard-agent-watch:${email}`, + }, + }); + } + + function listRequest(chatId: string) { + return { + request: new Request( + `https://app.trigger.dev/api/v1/dashboard-agent/alerts?chatId=${chatId}`, + { headers: { Authorization: "Bearer tr_uat_test" } } + ), + params: {}, + context: {} as never, + } as never; + } + + function createRequest(body: Record) { + return { + request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/alerts", { + method: "POST", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify(body), + }), + params: {}, + context: {} as never, + } as never; + } + + function deleteRequest(channelId: string, body: Record) { + return { + request: new Request(`https://app.trigger.dev/api/v1/dashboard-agent/alerts/${channelId}`, { + method: "DELETE", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify(body), + }), + params: { channelId }, + context: {} as never, + } as never; + } + + postgresTest( + "the dashboard lets any organization member manage a project's alerts", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-policy"); + const member = await seedMember(prisma, seeded); + const outsider = await seedOutsider(prisma); + + // The whole of the Alerts page's authorization, for list, create and delete alike. + expect( + await findProjectBySlug(seeded.organization.slug, seeded.project.slug, member.id) + ).not.toBeNull(); + expect( + await findProjectBySlug(seeded.organization.slug, seeded.project.slug, outsider.id) + ).toBeNull(); + } + ); + + postgresTest( + "a plain member reads and writes watch alerts through the agent, an outsider reads nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-member"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + await seedWatchChannel(prisma, seeded, member.email); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const listed = (await alertsLoader(listRequest("chat_member"))) as Response; + expect(listed.status).toBe(200); + // The same channel the Alerts page would show this member. + expect((await listed.json()).alerts).toHaveLength(1); + + // An outsider has no chat here and no membership, so nothing resolves. + ctx.actor = { + userId: (await seedOutsider(prisma)).id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const refused = (await alertsLoader(listRequest("chat_member"))) as Response; + expect(refused.status).toBe(404); + } + ); + + postgresTest( + "the agent only ever subscribes the caller's own address", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-create"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const own = (await alertsAction( + createRequest({ chatId: "chat_member", channel: "email" }) + )) as Response; + expect(own.status).toBe(200); + expect((await own.json()).target).toBe(member.email); + + // The Alerts page would let this member add anyone; the agent may not. + const other = (await alertsAction( + createRequest({ + chatId: "chat_member", + channel: "email", + email: "someone-else@example.com", + }) + )) as Response; + expect(other.status).toBe(400); + expect(await other.json()).toMatchObject({ code: "email_not_allowed" }); + + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(1); + } + ); + + postgresTest( + "the agent's delete only takes the watch type off a watch channel", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-delete"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + const watchChannel = await seedWatchChannel(prisma, seeded, member.email); + + // A channel the agent never created and has no business touching. + const runAlerts = await prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, + name: "Run failures", + projectId: seeded.project.id, + alertTypes: ["TASK_RUN"], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email: member.email }, + }, + }); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const removed = (await alertChannelAction( + deleteRequest(watchChannel.id, { chatId: "chat_member" }) + )) as Response; + expect(removed.status).toBe(200); + expect(await removed.json()).toMatchObject({ ok: true, disabledChannel: true }); + + // The Alerts page would let a member delete this outright; the agent gets a 404. + const untouched = (await alertChannelAction( + deleteRequest(runAlerts.id, { chatId: "chat_member" }) + )) as Response; + expect(untouched.status).toBe(404); + expect( + await prisma.projectAlertChannel.findFirst({ where: { id: runAlerts.id } }) + ).toMatchObject({ enabled: true, alertTypes: ["TASK_RUN"] }); + + // An outsider can't reach the channel at all. + ctx.actor = { + userId: (await seedOutsider(prisma)).id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const refused = (await alertChannelAction( + deleteRequest(watchChannel.id, { chatId: "chat_member" }) + )) as Response; + expect(refused.status).toBe(404); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.routes.test.ts b/apps/webapp/test/dashboardAgentWatches.routes.test.ts deleted file mode 100644 index 74f9fd86b97..00000000000 --- a/apps/webapp/test/dashboardAgentWatches.routes.test.ts +++ /dev/null @@ -1,731 +0,0 @@ -import { - createChat, - getWatch, - listActiveWatchesForChat, - recordWatchCheck, - type DashboardAgentDb, -} from "@internal/dashboard-agent-db"; -import type { WatchSpec } from "@internal/dashboard-agent-contracts"; -import { postgresTest } from "@internal/testcontainers"; -import type { PrismaClient } from "@trigger.dev/database"; -import { afterEach, beforeEach, describe, expect, vi } from "vitest"; -import { previousCheckFacts } from "~/services/dashboardAgentWatchChecks"; -import { - DashboardAgentWatchesTestHarness, - RUN_START, - type DashboardAgentWatchesTestContext, - type Seeded, -} from "./helpers/dashboardAgentWatchesTestHelpers"; - -vi.setConfig({ testTimeout: 60_000 }); - -const ctx = vi.hoisted( - (): DashboardAgentWatchesTestContext => ({ - prisma: undefined as unknown as PrismaClient, - agentDb: undefined as unknown as DashboardAgentDb, - canAccess: true, - actor: undefined, - triggered: [], - }) -); - -vi.mock("~/services/uatRoutePreamble.server", () => ({ - authenticateUatOrApiRequest: async () => - ctx.actor - ? { - authenticationResult: { - type: "personalAccessToken", - result: { userId: ctx.actor.userId }, - }, - userActor: ctx.actor, - } - : undefined, -})); - -vi.mock("~/db.server", () => { - const proxy = new Proxy( - {}, - { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } - ); - return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; -}); - -vi.mock("~/services/dashboardAgentDb.server", () => ({ - get dashboardAgentDb() { - return ctx.agentDb; - }, -})); - -vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ - canAccessDashboardAgent: async () => ctx.canAccess, -})); - -vi.mock("@trigger.dev/sdk", async (importOriginal) => { - const actual = await importOriginal>(); - return { - ...actual, - TriggerClient: class { - tasks = { - trigger: async (taskId: string) => { - ctx.triggered.push(taskId); - return { id: "run_test" }; - }, - }; - }, - }; -}); - -const SESSION_SECRET = "test-session-secret-for-watch-tokens"; -process.env.SESSION_SECRET = SESSION_SECRET; -process.env.ALERT_FROM_EMAIL = "alerts@example.com"; -process.env.ALERT_EMAIL_TRANSPORT = "smtp"; -process.env.DASHBOARD_AGENT_SECRET_KEY = "test-dashboard-agent-secret"; - -const { createDashboardAgentWatch } = await import("~/services/dashboardAgentWatches.server"); -const { action: checkAction } = - await import("~/routes/api.v1.dashboard-agent.watches.$watchId.check"); -const { action: createAction } = await import("~/routes/api.v1.dashboard-agent.watches"); -const { signDashboardAgentWatchToken } = await import("~/services/dashboardAgentWatchToken.server"); -const { loader: alertsLoader, action: alertsAction } = - await import("~/routes/api.v1.dashboard-agent.alerts"); -const { action: alertChannelAction } = - await import("~/routes/api.v1.dashboard-agent.alerts.$channelId"); -const { findProjectBySlug } = await import("~/models/project.server"); -const { DASHBOARD_AGENT_WATCH_ALERT_TYPE } = - await import("~/services/dashboardAgentWatchAlerts.server"); - -const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); -const boot = harness.boot.bind(harness); -const seed = harness.seed.bind(harness); -const seedChat = harness.seedChat.bind(harness); -const create = harness.create.bind(harness); - -beforeEach(() => harness.reset()); -afterEach(() => harness.close()); - -describe("the createWatch endpoint's authorization", () => { - function post(body: unknown) { - return createAction({ - request: new Request("https://example.com/api/v1/dashboard-agent/watches", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(body), - }), - params: {}, - context: {}, - }); - } - - const validBody = (chatId: string) => ({ spec: RUN_START, chatId }); - - postgresTest("401s without a delegated token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const response = await post(validBody("chat_1")); - expect(response.status).toBe(401); - }); - - postgresTest("403s for any other client's token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "adapter"); - ctx.actor = { userId: seeded.user.id, client: "cli", environmentId: seeded.environment.id }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "forbidden_client" }); - }); - - postgresTest( - "refuses a chat the authenticated user doesn't own, writing nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const owner = await seed(prisma, "owner"); - const stranger = await seed(prisma, "stranger"); - await createChat(ctx.agentDb, { - id: "chat_victim", - organizationId: owner.organization.id, - userId: owner.user.id, - }); - - ctx.actor = { - userId: stranger.user.id, - client: "dashboard-agent", - environmentId: stranger.environment.id, - }; - - const response = await post(validBody("chat_victim")); - expect(response.status).toBe(404); - expect(await response.json()).toMatchObject({ code: "chat_not_found" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_victim" })).toHaveLength( - 0 - ); - } - ); - - postgresTest( - "refuses a token with no environment scope", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "noscope"); - await seedChat(seeded, "chat_1"); - ctx.actor = { userId: seeded.user.id, client: "dashboard-agent" }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "invalid_target" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "refuses a body naming a different environment than the token's", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "mismatch"); - const other = await seed(prisma, "othermismatch"); - await seedChat(seeded, "chat_1"); - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const response = await post({ - ...validBody("chat_1"), - environmentId: other.environment.id, - }); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "binds to the token's environment, not the chat's stored context", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "binding"); - const otherProject = await prisma.project.create({ - data: { - name: `${seeded.project.slug}_b`, - slug: `${seeded.project.slug}_b`, - organizationId: seeded.organization.id, - externalRef: `proj_${seeded.project.slug}_b`, - }, - }); - const otherEnvironment = await prisma.runtimeEnvironment.create({ - data: { - slug: "prod", - type: "PRODUCTION", - projectId: otherProject.id, - organizationId: seeded.organization.id, - apiKey: `tr_prod_${otherProject.slug}`, - pkApiKey: `pk_prod_${otherProject.slug}`, - shortcode: `b${otherProject.slug.slice(0, 6)}`, - }, - }); - - await createChat(ctx.agentDb, { - id: "chat_1", - organizationId: seeded.organization.id, - userId: seeded.user.id, - metadata: { - context: { - environmentId: seeded.environment.id, - projectRef: seeded.project.externalRef, - }, - }, - }); - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: otherEnvironment.id, - }; - - const response = await post({ - ...validBody("chat_1"), - projectRef: seeded.project.externalRef, - }); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "refuses an environment in another org than the chat's", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "crossorg"); - const other = await seed(prisma, "otherorg"); - await prisma.orgMember.create({ - data: { - organizationId: other.organization.id, - userId: seeded.user.id, - role: "ADMIN", - }, - }); - await seedChat(seeded, "chat_1"); - - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: other.environment.id, - }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(404); - expect(await response.json()).toMatchObject({ code: "invalid_target" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); -}); - -describe("the check endpoint", () => { - function request(token: string, body: unknown = {}) { - return new Request("https://example.com/api/v1/dashboard-agent/watches/x/check", { - method: "POST", - headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, - body: JSON.stringify(body), - }); - } - - async function activeWatch(seeded: Seeded, spec?: WatchSpec) { - const result = await create({ seeded, spec }); - if (!result.ok) throw new Error(`watch not created: ${result.code}`); - return result; - } - - function tokenFor(watchId: string, expiresAt: Date) { - return signDashboardAgentWatchToken(SESSION_SECRET, { watchId, expiresAt }); - } - - postgresTest("401s on a bad token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - - const response = await checkAction({ - request: request("tr_daw_nonsense"), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(401); - }); - - postgresTest("403s when the token names another watch", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor("watch_someone_else", watch.expiresAt); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "watch_mismatch" }); - }); - - postgresTest("answers a check and records what it saw", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(200); - const body = await response.json(); - expect(body.result).toBe("terminal_unsatisfied"); - - // Arming the chain goes through the stubbed client, never a real trigger. - expect(ctx.triggered).toContain("dashboard-agent-watch-batch"); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - expect(row?.lastCheckedAt).not.toBeNull(); - expect(row?.tickCount).toBe(0); - expect(row?.status).toBe("active"); - }); - - postgresTest( - "refuses an ordinary check after expiry but allows the final one in grace", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - - await prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 minute' where id = $1`, - watch.watchId - ); - - const token = await tokenFor(watch.watchId, watch.expiresAt); - - const refused = await checkAction({ - request: request(token, {}), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(refused.status).toBe(403); - expect(await refused.json()).toMatchObject({ code: "expired" }); - - const allowed = await checkAction({ - request: request(token, { final: true }), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(allowed.status).toBe(200); - } - ); - - postgresTest( - "cancels the watch on revoked access, without reading environment data", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "access_revoked" }); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - expect(row).toMatchObject({ - status: "cancelled", - cancelReason: "access_revoked", - deliveryStatus: "not_required", - }); - expect(row?.tickCount).toBe(0); - expect(row?.lastResult).toBeNull(); - } - ); - - postgresTest( - "a check that couldn't read anything leaves the row's last look and facts alone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - - // The queue exists, so the check gets past the target read and fails on the depth - // read: there is no live queue or analytics store behind this environment. - const queue = "task/stalling"; - await prisma.taskQueue.create({ - data: { - runtimeEnvironmentId: seeded.environment.id, - projectId: seeded.project.id, - name: queue, - friendlyId: `queue_${Math.random().toString(36).slice(2, 10)}`, - orderableName: queue, - }, - }); - - const watch = await activeWatch(seeded, { - kind: "queue_stalled", - queue, - ticks: 3, - checkEveryMinutes: 5, - maxHours: 6, - note: "tell me if the queue stops moving", - }); - - // Two no-progress checks already behind it, last looked at an hour ago. - const checkedAt = new Date(Date.now() - 60 * 60 * 1000); - await recordWatchCheck(ctx.agentDb, { - id: watch.watchId, - lastCheckedAt: checkedAt, - lastResult: { - result: "pending", - facts: { queue, depth: 412, notDecreasingStreak: 2, ticks: 3 }, - }, - }); - - const token = await tokenFor(watch.watchId, watch.expiresAt); - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ result: "unavailable" }); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - // Nothing was checked, so the watch is still due at the next tick. - expect(row?.lastCheckedAt?.getTime()).toBe(checkedAt.getTime()); - // And the streak the earlier ticks built is still there to be continued. - expect(previousCheckFacts(row?.lastResult)).toMatchObject({ - depth: 412, - notDecreasingStreak: 2, - }); - }, - 120_000 - ); - - postgresTest("403s once the watch is terminal", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - await prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set status = 'cancelled' where id = $1`, - watch.watchId - ); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "cancelled" }); - }); -}); - -describe("the agent's alert boundary", () => { - /** A second, plain member of the same organization. */ - async function seedMember(prisma: PrismaClient, seeded: Seeded) { - const member = await prisma.user.create({ - data: { - email: `member_${Math.random().toString(36).slice(2, 10)}@example.com`, - authenticationMethod: "MAGIC_LINK", - }, - }); - await prisma.orgMember.create({ - data: { organizationId: seeded.organization.id, userId: member.id, role: "MEMBER" }, - }); - return member; - } - - async function seedOutsider(prisma: PrismaClient) { - return prisma.user.create({ - data: { - email: `outsider_${Math.random().toString(36).slice(2, 10)}@example.com`, - authenticationMethod: "MAGIC_LINK", - }, - }); - } - - async function seedWatchChannel(prisma: PrismaClient, seeded: Seeded, email: string) { - return prisma.projectAlertChannel.create({ - data: { - friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, - name: `Watch alerts for ${email}`, - projectId: seeded.project.id, - alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE as never], - environmentTypes: ["PRODUCTION"], - type: "EMAIL", - properties: { email }, - deduplicationKey: `dashboard-agent-watch:${email}`, - }, - }); - } - - function listRequest(chatId: string) { - return { - request: new Request( - `https://app.trigger.dev/api/v1/dashboard-agent/alerts?chatId=${chatId}`, - { headers: { Authorization: "Bearer tr_uat_test" } } - ), - params: {}, - context: {} as never, - } as never; - } - - function createRequest(body: Record) { - return { - request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/alerts", { - method: "POST", - headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, - body: JSON.stringify(body), - }), - params: {}, - context: {} as never, - } as never; - } - - function deleteRequest(channelId: string, body: Record) { - return { - request: new Request(`https://app.trigger.dev/api/v1/dashboard-agent/alerts/${channelId}`, { - method: "DELETE", - headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, - body: JSON.stringify(body), - }), - params: { channelId }, - context: {} as never, - } as never; - } - - postgresTest( - "the dashboard lets any organization member manage a project's alerts", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-policy"); - const member = await seedMember(prisma, seeded); - const outsider = await seedOutsider(prisma); - - // The whole of the Alerts page's authorization, for list, create and delete alike. - expect( - await findProjectBySlug(seeded.organization.slug, seeded.project.slug, member.id) - ).not.toBeNull(); - expect( - await findProjectBySlug(seeded.organization.slug, seeded.project.slug, outsider.id) - ).toBeNull(); - } - ); - - postgresTest( - "a plain member reads and writes watch alerts through the agent, an outsider reads nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-member"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - await seedWatchChannel(prisma, seeded, member.email); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const listed = (await alertsLoader(listRequest("chat_member"))) as Response; - expect(listed.status).toBe(200); - // The same channel the Alerts page would show this member. - expect((await listed.json()).alerts).toHaveLength(1); - - // An outsider has no chat here and no membership, so nothing resolves. - ctx.actor = { - userId: (await seedOutsider(prisma)).id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const refused = (await alertsLoader(listRequest("chat_member"))) as Response; - expect(refused.status).toBe(404); - } - ); - - postgresTest( - "the agent only ever subscribes the caller's own address", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-create"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const own = (await alertsAction( - createRequest({ chatId: "chat_member", channel: "email" }) - )) as Response; - expect(own.status).toBe(200); - expect((await own.json()).target).toBe(member.email); - - // The Alerts page would let this member add anyone; the agent may not. - const other = (await alertsAction( - createRequest({ - chatId: "chat_member", - channel: "email", - email: "someone-else@example.com", - }) - )) as Response; - expect(other.status).toBe(400); - expect(await other.json()).toMatchObject({ code: "email_not_allowed" }); - - expect( - await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) - ).toBe(1); - } - ); - - postgresTest( - "the agent's delete only takes the watch type off a watch channel", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-delete"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - const watchChannel = await seedWatchChannel(prisma, seeded, member.email); - - // A channel the agent never created and has no business touching. - const runAlerts = await prisma.projectAlertChannel.create({ - data: { - friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, - name: "Run failures", - projectId: seeded.project.id, - alertTypes: ["TASK_RUN"], - environmentTypes: ["PRODUCTION"], - type: "EMAIL", - properties: { email: member.email }, - }, - }); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const removed = (await alertChannelAction( - deleteRequest(watchChannel.id, { chatId: "chat_member" }) - )) as Response; - expect(removed.status).toBe(200); - expect(await removed.json()).toMatchObject({ ok: true, disabledChannel: true }); - - // The Alerts page would let a member delete this outright; the agent gets a 404. - const untouched = (await alertChannelAction( - deleteRequest(runAlerts.id, { chatId: "chat_member" }) - )) as Response; - expect(untouched.status).toBe(404); - expect( - await prisma.projectAlertChannel.findFirst({ where: { id: runAlerts.id } }) - ).toMatchObject({ enabled: true, alertTypes: ["TASK_RUN"] }); - - // An outsider can't reach the channel at all. - ctx.actor = { - userId: (await seedOutsider(prisma)).id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const refused = (await alertChannelAction( - deleteRequest(watchChannel.id, { chatId: "chat_member" }) - )) as Response; - expect(refused.status).toBe(404); - } - ); -}); diff --git a/apps/webapp/test/dashboardAgentWatches.submit.test.ts b/apps/webapp/test/dashboardAgentWatches.submit.test.ts deleted file mode 100644 index cc0109efe57..00000000000 --- a/apps/webapp/test/dashboardAgentWatches.submit.test.ts +++ /dev/null @@ -1,617 +0,0 @@ -import { - appendChatMessageOnce, - countUserMessages, - getWatch, - getWatchSubmission, - listActiveWatchesForChat, - recordWatchSubmissionOutcome, - transitionWatchCondition, - type DashboardAgentDb, -} from "@internal/dashboard-agent-db"; -import type { WatchDraft } from "@internal/dashboard-agent-contracts"; -import { postgresTest } from "@internal/testcontainers"; -import type { PrismaClient } from "@trigger.dev/database"; -import { afterEach, beforeEach, describe, expect, vi } from "vitest"; -import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; -import { - DashboardAgentWatchesTestHarness, - RUN_START, - draftFor, - type DashboardAgentWatchesTestContext, - type Seeded, -} from "./helpers/dashboardAgentWatchesTestHelpers"; - -vi.setConfig({ testTimeout: 60_000 }); - -const ctx = vi.hoisted( - (): DashboardAgentWatchesTestContext => ({ - prisma: undefined as unknown as PrismaClient, - agentDb: undefined as unknown as DashboardAgentDb, - canAccess: true, - actor: undefined, - triggered: [], - }) -); - -vi.mock("~/db.server", () => { - const proxy = new Proxy( - {}, - { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } - ); - return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; -}); - -vi.mock("~/services/dashboardAgentDb.server", () => ({ - get dashboardAgentDb() { - return ctx.agentDb; - }, -})); - -process.env.SESSION_SECRET = "test-session-secret-for-watch-tokens"; -process.env.ALERT_FROM_EMAIL = "alerts@example.com"; -process.env.ALERT_EMAIL_TRANSPORT = "smtp"; - -const { createDashboardAgentWatch, submitDashboardAgentWatch } = - await import("~/services/dashboardAgentWatches.server"); -const { subscribeUserToWatchAlerts } = await import("~/services/dashboardAgentWatchAlerts.server"); - -const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); -const boot = harness.boot.bind(harness); -const seed = harness.seed.bind(harness); -const authenticated = harness.authenticated.bind(harness); -const seedChat = harness.seedChat.bind(harness); -const runRow = harness.runRow.bind(harness); -const fakeCheckDeps = harness.fakeCheckDeps.bind(harness); -const create = harness.create.bind(harness); -const storedMessages = harness.storedMessages.bind(harness); - -beforeEach(() => harness.reset()); -afterEach(() => harness.close()); - -function submit(args: { - seeded: Seeded; - draft?: WatchDraft; - chatId?: string; - clientRequestId?: string; - checkDeps?: Partial; - subscribed?: boolean; - /** Replaces the fake outright, so a test can hand the submit the real subscribe. */ - subscribe?: typeof subscribeUserToWatchAlerts; - onSchedule?: () => void; - /** Wraps the creation step, so a test can die at the exact point after it. */ - create?: typeof createDashboardAgentWatch; -}) { - return submitDashboardAgentWatch({ - environment: authenticated(args.seeded), - userId: args.seeded.user.id, - organizationId: args.seeded.organization.id, - chatId: args.chatId, - clientRequestId: args.clientRequestId ?? "wreq_1", - draft: args.draft ?? draftFor(RUN_START), - deps: { - configured: () => true, - checkDeps: () => fakeCheckDeps(args.checkDeps), - scheduleTick: async () => args.onSchedule?.(), - ...(args.create ? { create: args.create } : {}), - subscribe: - args.subscribe ?? - (async () => - args.subscribed === false - ? { ok: false, reason: "dashboard_agent_disabled" } - : { ok: true, email: args.seeded.user.email }), - }, - }); -} - -describe("the watch card submit", () => { - postgresTest( - "records what the user confirmed before the watch, and confirms it after", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit"); - await seedChat(seeded); - - const result = await submit({ - seeded, - chatId: "chat_1", - draft: draftFor(RUN_START, { investigateOnAttention: true }), - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(true); - expect(result.repaired).toBe(false); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${result.watchId}`, - ]); - // The consent record is the user's, and it states the condition and the lifetime. - expect(stored?.[0]).toMatchObject({ role: "user" }); - expect(JSON.stringify(stored?.[0])).toContain("Watch run run_1 until it starts."); - expect(JSON.stringify(stored?.[0])).toContain("Investigate straight away"); - expect(result.messages.map((message) => message.id)).toEqual( - stored?.map((message) => message.id) - ); - } - ); - - postgresTest( - "leaves a repairable state when the confirmation never lands, and the retry repairs it", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-repair"); - await seedChat(seeded); - - // The crash state: the request record is written and the watch is live, but the - // process died before the confirmation was appended. - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - message: { id: "watch-request:wreq_1", role: "user", parts: [] } as never, - }); - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - if (!created.ok || !created.watching) return; - - const retry = await submit({ seeded, chatId: "chat_1", clientRequestId: "wreq_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(created.watchId); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${created.watchId}`, - ]); - - // Still exactly one watch: the repair loaded it rather than creating another. - const active = await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" }); - expect(active).toHaveLength(1); - } - ); - - postgresTest( - "a retried submit duplicates neither record", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-retry"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - const second = await submit({ seeded, chatId: "chat_1" }); - - expect(first.ok && second.ok).toBe(true); - if (!first.ok || !second.ok) return; - expect(second.repaired).toBe(true); - expect(second.watchId).toBe(first.watchId); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a genuinely different request still conflicts", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-conflict"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok) return; - - // Same condition, so the same identity, but a different window: not a retry. - const longer = await submit({ - seeded, - chatId: "chat_1", - clientRequestId: "wreq_2", - draft: draftFor({ ...RUN_START, maxHours: 6 }), - }); - expect(longer).toMatchObject({ ok: false, code: "duplicate", existingId: first.watchId }); - - // Same spec, different consent: also not a retry. - const investigating = await submit({ - seeded, - chatId: "chat_1", - clientRequestId: "wreq_3", - draft: draftFor(RUN_START, { investigateOnAttention: true }), - }); - expect(investigating).toMatchObject({ ok: false, code: "duplicate" }); - - // The refused attempts are recorded under their own consent records, so the - // transcript never shows a request with no answer. - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - "watch-request:wreq_2", - "watch-confirmation:refused:wreq_2", - "watch-request:wreq_3", - "watch-confirmation:refused:wreq_3", - ]); - } - ); - - postgresTest( - "a fresh panel's retry reuses the chat the first attempt created", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-fresh"); - - const first = await submit({ seeded, clientRequestId: "wreq_fresh" }); - const second = await submit({ seeded, clientRequestId: "wreq_fresh" }); - - expect(first.ok && second.ok).toBe(true); - if (!first.ok || !second.ok) return; - expect(second.chatId).toBe(first.chatId); - - const stored = await storedMessages(seeded, first.chatId); - expect(stored).toHaveLength(2); - } - ); - - postgresTest( - "an answered condition records the request and a one-shot result, and never a watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-oneshot"); - await seedChat(seeded); - - const result = await submit({ - seeded, - chatId: "chat_1", - checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(false); - expect(result.watchId).toBeNull(); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - "watch-confirmation:one-shot:wreq_1", - ]); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - /** Every watch row for a chat, terminal ones included. `listActiveWatchesForChat` can't see those. */ - async function countWatchRows(prisma: PrismaClient, chatId: string) { - const rows = await prisma.$queryRawUnsafe>( - `select count(*)::bigint as count from trigger_dashboard_agent.watches where chat_id = $1`, - chatId - ); - return Number(rows[0]?.count ?? 0); - } - - postgresTest( - "a retry after the watch has already fired creates no second watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-fired"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok || !first.watchId) return; - - // The watch resolves and leaves the active set, so a duplicate check would find - // nothing. Only the ledger still knows this request already ran. - await transitionWatchCondition(ctx.agentDb, { - id: first.watchId, - resolution: "condition_met", - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(first.watchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a retry of an answered one-shot never becomes a watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-oneshot-retry"); - await seedChat(seeded); - - const first = await submit({ - seeded, - chatId: "chat_1", - checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, - }); - expect(first.ok && first.watching === false).toBe(true); - - // The world moved on: the same condition would now be pending, so a re-evaluation - // would start a real watch. The recorded outcome is replayed instead. - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.watching).toBe(false); - expect(retry.watchId).toBeNull(); - expect(retry.repaired).toBe(true); - expect(await countWatchRows(prisma, "chat_1")).toBe(0); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - "watch-confirmation:one-shot:wreq_1", - ]); - } - ); - - postgresTest( - "the same request id carrying a different draft is a conflict", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-hash"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok) return; - - const changed = await submit({ - seeded, - chatId: "chat_1", - draft: draftFor({ ...RUN_START, maxHours: 6 }), - }); - expect(changed).toMatchObject({ ok: false, code: "request_conflict" }); - - // A conflict writes nothing at all: no watch, and no record under the request. - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a pending submission converges on the watch its first attempt created", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-converge"); - await seedChat(seeded); - - // The crash state the ledger exists for: the row is reserved, the watch is live - // under the reserved id, and the process died before the outcome was written. - let reservedWatchId = ""; - await expect( - submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - await createDashboardAgentWatch(createParams); - throw new Error("died after the watch was created"); - }, - }) - ).rejects.toThrow("died after the watch was created"); - - const pending = await getWatchSubmission(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - }); - expect(pending).toMatchObject({ state: "pending", watchId: reservedWatchId }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - // Reached the reserved row rather than creating another. - expect(retry.watchId).toBe(reservedWatchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const settled = await getWatchSubmission(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - }); - expect(settled).toMatchObject({ state: "created", watchId: reservedWatchId }); - } - ); - - postgresTest( - "converging on a watch that already fired confirms the outcome, not 'watching'", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-converge-fired"); - await seedChat(seeded); - - let reservedWatchId = ""; - await expect( - submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - await createDashboardAgentWatch(createParams); - throw new Error("died after the watch was created"); - }, - }) - ).rejects.toThrow("died after the watch was created"); - - // The watch ran and woke the chat before anyone retried the submit. - await transitionWatchCondition(ctx.agentDb, { - id: reservedWatchId, - resolution: "condition_met", - observedOutcome: { kind: "run_start", verified: true, status: "EXECUTING", started: true }, - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - // Still one row, still the same watch: adoption is not refused. - expect(retry.watchId).toBe(reservedWatchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const parts = retry.messages.at(-1)?.parts ?? []; - const block = (parts[0] as any).data.blocks[0]; - expect(block.outcome).toBe("already_true"); - expect(block.headline).not.toContain("Watching"); - expect(block.lifetime).toBeNull(); - } - ); - - postgresTest( - "a refusal that wins the race leaves no live watch behind", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-refused-race"); - await seedChat(seeded); - - // A concurrent attempt refuses this submission after the watch exists under the - // reserved id, so the ledger's winner keeps naming that id. - let reservedWatchId = ""; - const result = await submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - const created = await createDashboardAgentWatch(createParams); - const refused = await recordWatchSubmissionOutcome(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - state: "refused", - refusalCode: "internal", - refusalError: "That watch couldn't be started.", - }); - expect(refused).toMatchObject({ state: "refused", watchId: reservedWatchId }); - return created; - }, - }); - - // The user is told nothing is being watched, so nothing may be watching. - expect(result.ok).toBe(false); - const row = await getWatch(ctx.agentDb, { id: reservedWatchId }); - expect(row).toMatchObject({ status: "cancelled", cancelReason: "superseded" }); - } - ); - - postgresTest( - "the consent record never spends a message from the cap", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-quota"); - await seedChat(seeded); - - await submit({ seeded, chatId: "chat_1" }); - - expect( - await countUserMessages(ctx.agentDb, { - organizationId: seeded.organization.id, - userId: seeded.user.id, - }) - ).toBe(0); - } - ); - - postgresTest( - "a replay repeats the recorded email outcome and subscribes nobody", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-external-replay"); - await seedChat(seeded); - - const draft = draftFor(RUN_START, { notifyExternally: true }); - - // The first attempt asked for email and couldn't get it, so `unavailable` is what - // the transcript says and what the ledger records. - const first = await submit({ seeded, chatId: "chat_1", draft, subscribed: false }); - expect(first.ok).toBe(true); - if (!first.ok) return; - expect(JSON.stringify(first.messages)).toContain("I couldn't add email notifications"); - expect( - await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) - ).toMatchObject({ state: "created", externalNotificationStatus: "unavailable" }); - - const transcript = await storedMessages(seeded, "chat_1"); - - // The retry gets the real subscribe, which would succeed here. A replay that took the - // decision again would leave a channel row and an `enabled` answer the transcript โ€” - // append-once, so never rewritten โ€” contradicts for good. - let subscribeCalls = 0; - const retry = await submit({ - seeded, - chatId: "chat_1", - draft, - subscribe: async (subscribeParams) => { - subscribeCalls++; - return subscribeUserToWatchAlerts(subscribeParams); - }, - }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(first.watchId); - expect(subscribeCalls).toBe(0); - - expect(JSON.stringify(retry.messages)).toContain("I couldn't add email notifications"); - expect(JSON.stringify(retry.messages)).not.toContain("You'll get an email"); - expect( - await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) - ).toBe(0); - expect( - await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) - ).toMatchObject({ externalNotificationStatus: "unavailable" }); - - // The symptom: what the user is told after a refresh has to agree with the answer. - expect(await storedMessages(seeded, "chat_1")).toEqual(transcript); - } - ); - - postgresTest( - "a replay repeats the recorded 'Watching' confirmation after the watch has fired", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-replay-fired"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok || !first.watchId) return; - - await transitionWatchCondition(ctx.agentDb, { - id: first.watchId, - resolution: "condition_met", - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - - // The recorded outcome is replayed, never decided again: the append-once - // confirmation in the transcript says "Watching", so the answer has to as well. - const parts = retry.messages.at(-1)?.parts ?? []; - const block = (parts[0] as any).data.blocks[0]; - expect(block.outcome).toBe("watching"); - expect(block.headline).toContain("Watching"); - } - ); -}); From 86a056731be70a41ab15795a23e572800b7145f8 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 17:24:30 +0100 Subject: [PATCH 09/15] perf(ci): recalibrate webapp timings after shutdown fixes --- test-timings.json | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/test-timings.json b/test-timings.json index 9397c0c667a..d088d9e9bea 100644 --- a/test-timings.json +++ b/test-timings.json @@ -65,10 +65,10 @@ "apps/webapp/app/routes/storybook.agent-ui/fixtures.test.ts": 6, "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 3, "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 2, - "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 50313, - "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 132163, - "apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts": 91190, - "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 132085, + "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 3318, + "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 7208, + "apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts": 2786, + "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 8861, "apps/webapp/app/utils/apiKeys.test.ts": 6, "apps/webapp/app/utils/boundedRequestBody.server.test.ts": 9, "apps/webapp/app/utils/cspImageOrigins.test.ts": 3, @@ -109,8 +109,8 @@ "apps/webapp/test/additionalApiKeyIssuance.test.ts": 3, "apps/webapp/test/aiTitleRateLimiter.test.ts": 55, "apps/webapp/test/api-auth.e2e.test.ts": 20090, - "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 94856, - "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 174422, + "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 30560, + "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 17241, "apps/webapp/test/apiAuthActorClaim.test.ts": 4, "apps/webapp/test/apiAuthScope.test.ts": 7, "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 16921, @@ -122,7 +122,8 @@ "apps/webapp/test/apiRateLimitJwtActor.test.ts": 4, "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 946, "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 8834, - "apps/webapp/test/apiRunListPresenter.test.ts": 141836, + "apps/webapp/test/apiRunListPresenter.readthrough.test.ts": 31646, + "apps/webapp/test/apiRunListPresenter.test.ts": 5271, "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 9048, "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 1713, "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 11841, @@ -231,7 +232,9 @@ "apps/webapp/test/dashboardAgentWatchTenancy.test.ts": 5475, "apps/webapp/test/dashboardAgentWatchToken.test.ts": 16, "apps/webapp/test/dashboardAgentWatchWording.test.ts": 7, - "apps/webapp/test/dashboardAgentWatches.test.ts": 123752, + "apps/webapp/test/dashboardAgentWatches.batch.test.ts": 34184, + "apps/webapp/test/dashboardAgentWatches.delivery.test.ts": 81151, + "apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts": 80861, "apps/webapp/test/deleteTaskSchedule.test.ts": 16973, "apps/webapp/test/deliveryIdBounds.test.ts": 18, "apps/webapp/test/dependentAttemptScope.test.ts": 2, @@ -250,17 +253,19 @@ "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 11035, "apps/webapp/test/engine/streamBatchItems.test.ts": 22941, "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 7090, - "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 132100, - "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 90077, - "apps/webapp/test/engine/triggerTask.debounce.test.ts": 174383, - "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133924, - "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 172695, - "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 172753, - "apps/webapp/test/engine/triggerTask.residency.test.ts": 172776, - "apps/webapp/test/engine/triggerTask.test.ts": 132169, + "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 17525, + "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 6851, + "apps/webapp/test/engine/triggerTask.debounce.test.ts": 12067, + "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 11663, + "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 7667, + "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 7963, + "apps/webapp/test/engine/triggerTask.residency.test.ts": 7611, + "apps/webapp/test/engine/triggerTask.test.ts": 10375, "apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts": 8985, "apps/webapp/test/env.server.test.ts": 500, - "apps/webapp/test/envConcurrencyLimitPause.server.test.ts": 214032, + "apps/webapp/test/envConcurrencyLimitPause.server.test.ts": 4829, + "apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts": 4856, + "apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts": 3077, "apps/webapp/test/envJwtActorClaim.test.ts": 20, "apps/webapp/test/envParamRoute.ownership.test.ts": 5, "apps/webapp/test/environmentSort.test.ts": 8, From 529eaa2b38f1e95ae929c23b2f726fb8dd9adf30 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 21:54:38 +0100 Subject: [PATCH 10/15] test(webapp): split external deployment trigger coverage --- ...riggerTask.externalDeploymentId.helpers.ts | 99 +++++ ...rTask.externalDeploymentId.pending.test.ts | 131 +++++++ ...sk.externalDeploymentId.resolution.test.ts | 182 +++++++++ .../triggerTask.externalDeploymentId.test.ts | 354 +----------------- 4 files changed, 417 insertions(+), 349 deletions(-) create mode 100644 apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts create mode 100644 apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts create mode 100644 apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts new file mode 100644 index 00000000000..ea1bb667c1d --- /dev/null +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts @@ -0,0 +1,99 @@ +import { RunEngine } from "@internal/run-engine"; +import { trace } from "@opentelemetry/api"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "ioredis"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import type { + ExternalDeploymentCache, + ExternalDeploymentCacheEntry, +} from "~/services/externalDeploymentCache.server"; +import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server"; +import { + MockPayloadProcessor, + MockTraceEventConcern, + MockTriggerTaskValidator, +} from "./triggerTaskTestHelpers"; + +export class RecordingExternalDeploymentCache implements ExternalDeploymentCache { + readonly gets: Array<{ environmentId: string; externalId: string }> = []; + readonly writes: Array<{ externalId: string; entry: ExternalDeploymentCacheEntry }> = []; + + constructor(private readonly entries = new Map()) {} + + readonly missing: string[] = []; + + async get(environmentId: string, externalId: string) { + this.gets.push({ environmentId, externalId }); + + const entry = this.entries.get(externalId); + + if (entry) { + return { outcome: "deployed" as const, entry }; + } + + return this.missing.includes(externalId) ? { outcome: "missing" as const } : null; + } + + async setIfNewer( + _environmentId: string, + externalId: string, + entry: ExternalDeploymentCacheEntry + ) { + this.writes.push({ externalId, entry }); + this.entries.set(externalId, entry); + } + + async setMissing(_environmentId: string, externalId: string) { + this.missing.push(externalId); + } +} + +export function createEngine(prisma: PrismaClient, redisOptions: RedisOptions) { + return new RunEngine({ + prisma, + worker: { redis: redisOptions, disabled: true }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + ttlSystem: { disabled: true }, + }, + batchQueue: { redis: redisOptions, consumerEnabled: false }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { name: "small-1x", cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} + +export function createService( + prisma: PrismaClient, + engine: RunEngine, + externalDeploymentCache: ExternalDeploymentCache +) { + return new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern(prisma, engine, new MockTraceEventConcern()), + validator: new MockTriggerTaskValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024, + externalDeploymentCache, + }); +} + +export async function nameDeploymentWithExternalId( + prisma: PrismaClient, + workerId: string, + externalId: string +) { + await prisma.workerDeployment.update({ where: { workerId }, data: { externalId } }); +} diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts new file mode 100644 index 00000000000..89026011f0b --- /dev/null +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, onTestFinished, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, +})); + +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { NoopExternalDeploymentCache } from "~/services/externalDeploymentCache.server"; +import { + createEngine, + createService, + RecordingExternalDeploymentCache, +} from "./triggerTask.externalDeploymentId.helpers"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +describe("triggerTask external deployment id", () => { + containerTest( + "parks a run whose id nothing holds, recording the id in annotations", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "parked-task"; + + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const service = createService(prisma, engine, new NoopExternalDeploymentCache()); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-unknown" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING_VERSION"); + expect(run.statusReason).toBe("EXTERNAL_DEPLOYMENT_PENDING"); + expect(run.lockedToVersionId).toBeNull(); + expect((run.annotations as Record).externalDeploymentId).toBe( + "commit-unknown" + ); + } + ); + + containerTest( + "never parks in development, where no deployment can ever hold the id", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "DEVELOPMENT"); + const taskIdentifier = "dev-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const cache = new RecordingExternalDeploymentCache(); + const service = createService(prisma, engine, cache); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-unknown" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING"); + expect(run.statusReason).toBeNull(); + expect(cache.gets).toEqual([]); + expect((run.annotations as Record).externalDeploymentId).toBe( + "commit-unknown" + ); + } + ); + + containerTest( + "parks a run whose id is held only by an in-flight deployment", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "inflight-task"; + + const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); + + await prisma.workerDeployment.update({ + where: { workerId: worker.worker.id }, + data: { externalId: "commit-building", status: "BUILDING" }, + }); + + const cache = new RecordingExternalDeploymentCache(); + const service = createService(prisma, engine, cache); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-building" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING_VERSION"); + expect(run.lockedToVersionId).toBeNull(); + + expect(cache.writes).toEqual([]); + } + ); +}); diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts new file mode 100644 index 00000000000..9b9216b15f9 --- /dev/null +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, onTestFinished, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, +})); + +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { NoopExternalDeploymentCache } from "~/services/externalDeploymentCache.server"; +import { + createEngine, + createService, + RecordingExternalDeploymentCache, +} from "./triggerTask.externalDeploymentId.helpers"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +describe("triggerTask external deployment id", () => { + containerTest( + "trusts a cache hit without querying Postgres", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "cached-pin-task"; + + const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); + + const cache = new RecordingExternalDeploymentCache( + new Map([ + [ + "commit-cached", + { + workerId: worker.worker.id, + version: worker.worker.version, + sdkVersion: "", + cliVersion: "", + }, + ], + ]) + ); + + const service = createService(prisma, engine, cache); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-cached" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING"); + expect(run.lockedToVersionId).toBe(worker.worker.id); + expect(cache.gets).toEqual([{ environmentId: environment.id, externalId: "commit-cached" }]); + expect(cache.writes).toEqual([]); + } + ); + + containerTest( + "resolves to the highest version when several deployed deployments hold the id", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "forced-task"; + + const older = await setupBackgroundWorker(engine, environment, taskIdentifier); + await prisma.backgroundWorker.update({ + where: { id: older.worker.id }, + data: { version: "20260807.9" }, + }); + await prisma.workerDeployment.update({ + where: { workerId: older.worker.id }, + data: { + externalId: "commit-forced", + version: "20260807.9", + shortCode: "short_code_20260807.9", + }, + }); + + const newer = await setupBackgroundWorker(engine, environment, taskIdentifier); + await prisma.backgroundWorker.update({ + where: { id: newer.worker.id }, + data: { version: "20260807.10" }, + }); + await prisma.workerDeployment.update({ + where: { workerId: newer.worker.id }, + data: { + externalId: "commit-forced", + version: "20260807.10", + shortCode: "short_code_20260807.10", + }, + }); + + const service = createService(prisma, engine, new NoopExternalDeploymentCache()); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-forced" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.lockedToVersionId).toBe(newer.worker.id); + expect(run.taskVersion).toBe("20260807.10"); + } + ); + + containerTest( + "an id is environment-scoped, so a deployment in another environment never resolves it", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "scoped-task"; + + const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); + + const otherEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: "staging-scoped", + type: "STAGING", + projectId: environment.project.id, + organizationId: environment.organization.id, + apiKey: "tr_stg_scoped", + pkApiKey: "pk_stg_scoped", + shortcode: "stg-scoped", + }, + }); + + await prisma.workerDeployment.create({ + data: { + friendlyId: "deployment_elsewhere", + contentHash: "hash", + shortCode: "sc_elsewhere", + version: worker.worker.version, + status: "DEPLOYED", + externalId: "commit-elsewhere", + projectId: environment.project.id, + environmentId: otherEnvironment.id, + }, + }); + + const service = createService(prisma, engine, new NoopExternalDeploymentCache()); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-elsewhere" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING_VERSION"); + expect(run.lockedToVersionId).toBeNull(); + } + ); +}); diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts index d6dba9ccdda..3fc33c68c89 100644 --- a/apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts @@ -17,110 +17,17 @@ vi.mock("~/services/platform.v3.server", async (importOriginal) => { }; }); -import { RunEngine } from "@internal/run-engine"; import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; import { assertNonNullable, containerTest } from "@internal/testcontainers"; -import { trace } from "@opentelemetry/api"; -import type { PrismaClient } from "@trigger.dev/database"; -import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; -import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; import { - type ExternalDeploymentCache, - type ExternalDeploymentCacheEntry, - NoopExternalDeploymentCache, -} from "~/services/externalDeploymentCache.server"; -import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server"; -import { - MockPayloadProcessor, - MockTraceEventConcern, - MockTriggerTaskValidator, -} from "./triggerTaskTestHelpers"; + createEngine, + createService, + nameDeploymentWithExternalId, + RecordingExternalDeploymentCache, +} from "./triggerTask.externalDeploymentId.helpers"; vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); -class RecordingExternalDeploymentCache implements ExternalDeploymentCache { - readonly gets: Array<{ environmentId: string; externalId: string }> = []; - readonly writes: Array<{ externalId: string; entry: ExternalDeploymentCacheEntry }> = []; - - constructor(private readonly entries = new Map()) {} - - readonly missing: string[] = []; - - async get(environmentId: string, externalId: string) { - this.gets.push({ environmentId, externalId }); - - const entry = this.entries.get(externalId); - - if (entry) { - return { outcome: "deployed" as const, entry }; - } - - return this.missing.includes(externalId) ? { outcome: "missing" as const } : null; - } - - async setIfNewer( - _environmentId: string, - externalId: string, - entry: ExternalDeploymentCacheEntry - ) { - this.writes.push({ externalId, entry }); - this.entries.set(externalId, entry); - } - - async setMissing(_environmentId: string, externalId: string) { - this.missing.push(externalId); - } -} - -function createEngine(prisma: PrismaClient, redisOptions: unknown) { - const engine = new RunEngine({ - prisma, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - worker: { redis: redisOptions as any, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - queue: { redis: redisOptions as any }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - runLock: { redis: redisOptions as any }, - machines: { - defaultMachine: "small-1x", - machines: { - "small-1x": { name: "small-1x", cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, - }, - baseCostInCents: 0.0005, - }, - tracer: trace.getTracer("test", "0.0.0"), - }); - - return engine; -} - -function createService( - prisma: PrismaClient, - engine: RunEngine, - externalDeploymentCache: ExternalDeploymentCache -) { - return new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern(prisma, engine, new MockTraceEventConcern()), - validator: new MockTriggerTaskValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024, - externalDeploymentCache, - }); -} - -async function nameDeploymentWithExternalId( - prisma: PrismaClient, - workerId: string, - externalId: string -) { - await prisma.workerDeployment.update({ where: { workerId }, data: { externalId } }); -} - describe("triggerTask external deployment id", () => { containerTest( "pins the run to the deployment holding the id, not to whatever is current", @@ -230,255 +137,4 @@ describe("triggerTask external deployment id", () => { expect(cache.gets).toEqual([]); } ); - - containerTest( - "parks a run whose id nothing holds, recording the id in annotations", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "parked-task"; - - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const service = createService(prisma, engine, new NoopExternalDeploymentCache()); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-unknown" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING_VERSION"); - expect(run.statusReason).toBe("EXTERNAL_DEPLOYMENT_PENDING"); - expect(run.lockedToVersionId).toBeNull(); - expect((run.annotations as Record).externalDeploymentId).toBe( - "commit-unknown" - ); - } - ); - - containerTest( - "never parks in development, where no deployment can ever hold the id", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "DEVELOPMENT"); - const taskIdentifier = "dev-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const cache = new RecordingExternalDeploymentCache(); - const service = createService(prisma, engine, cache); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-unknown" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING"); - expect(run.statusReason).toBeNull(); - expect(cache.gets).toEqual([]); - expect((run.annotations as Record).externalDeploymentId).toBe( - "commit-unknown" - ); - } - ); - - containerTest( - "parks a run whose id is held only by an in-flight deployment", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "inflight-task"; - - const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); - - await prisma.workerDeployment.update({ - where: { workerId: worker.worker.id }, - data: { externalId: "commit-building", status: "BUILDING" }, - }); - - const cache = new RecordingExternalDeploymentCache(); - const service = createService(prisma, engine, cache); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-building" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING_VERSION"); - expect(run.lockedToVersionId).toBeNull(); - - expect(cache.writes).toEqual([]); - } - ); - - containerTest( - "trusts a cache hit without querying Postgres", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "cached-pin-task"; - - const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); - - const cache = new RecordingExternalDeploymentCache( - new Map([ - [ - "commit-cached", - { - workerId: worker.worker.id, - version: worker.worker.version, - sdkVersion: "", - cliVersion: "", - }, - ], - ]) - ); - - const service = createService(prisma, engine, cache); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-cached" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING"); - expect(run.lockedToVersionId).toBe(worker.worker.id); - expect(cache.gets).toEqual([{ environmentId: environment.id, externalId: "commit-cached" }]); - expect(cache.writes).toEqual([]); - } - ); - - containerTest( - "resolves to the highest version when several deployed deployments hold the id", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "forced-task"; - - const older = await setupBackgroundWorker(engine, environment, taskIdentifier); - await prisma.backgroundWorker.update({ - where: { id: older.worker.id }, - data: { version: "20260807.9" }, - }); - await prisma.workerDeployment.update({ - where: { workerId: older.worker.id }, - data: { - externalId: "commit-forced", - version: "20260807.9", - shortCode: "short_code_20260807.9", - }, - }); - - const newer = await setupBackgroundWorker(engine, environment, taskIdentifier); - await prisma.backgroundWorker.update({ - where: { id: newer.worker.id }, - data: { version: "20260807.10" }, - }); - await prisma.workerDeployment.update({ - where: { workerId: newer.worker.id }, - data: { - externalId: "commit-forced", - version: "20260807.10", - shortCode: "short_code_20260807.10", - }, - }); - - const service = createService(prisma, engine, new NoopExternalDeploymentCache()); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-forced" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.lockedToVersionId).toBe(newer.worker.id); - expect(run.taskVersion).toBe("20260807.10"); - } - ); - - containerTest( - "an id is environment-scoped, so a deployment in another environment never resolves it", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "scoped-task"; - - const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); - - const otherEnvironment = await prisma.runtimeEnvironment.create({ - data: { - slug: "staging-scoped", - type: "STAGING", - projectId: environment.project.id, - organizationId: environment.organization.id, - apiKey: "tr_stg_scoped", - pkApiKey: "pk_stg_scoped", - shortcode: "stg-scoped", - }, - }); - - await prisma.workerDeployment.create({ - data: { - friendlyId: "deployment_elsewhere", - contentHash: "hash", - shortCode: "sc_elsewhere", - version: worker.worker.version, - status: "DEPLOYED", - externalId: "commit-elsewhere", - projectId: environment.project.id, - environmentId: otherEnvironment.id, - }, - }); - - const service = createService(prisma, engine, new NoopExternalDeploymentCache()); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-elsewhere" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING_VERSION"); - expect(run.lockedToVersionId).toBeNull(); - } - ); }); From 315241e6f12fba6e9562d6273b8363ee4fb7499f Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 22:00:36 +0100 Subject: [PATCH 11/15] perf(ci): rebalance webapp shards with CI timings --- test-timings.json | 931 +++++++++++++++++++++++----------------------- 1 file changed, 469 insertions(+), 462 deletions(-) diff --git a/test-timings.json b/test-timings.json index d088d9e9bea..7f45aa17099 100644 --- a/test-timings.json +++ b/test-timings.json @@ -1,537 +1,544 @@ { - "apps/webapp/app/components/code/StreamdownRenderer.test.ts": 284, - "apps/webapp/app/components/code/tsql/tsqlLinter.test.ts": 119, - "apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.render.test.ts": 21, - "apps/webapp/app/components/dashboard-agent/InvestigationCard.render.test.ts": 14, - "apps/webapp/app/components/dashboard-agent/InvestigationCard.test.ts": 6, + "apps/webapp/app/components/code/StreamdownRenderer.test.ts": 172, + "apps/webapp/app/components/code/tsql/tsqlLinter.test.ts": 177, + "apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.render.test.ts": 24, + "apps/webapp/app/components/dashboard-agent/InvestigationCard.render.test.ts": 8, + "apps/webapp/app/components/dashboard-agent/InvestigationCard.test.ts": 4, "apps/webapp/app/components/dashboard-agent/ReportView.test.ts": 2, "apps/webapp/app/components/dashboard-agent/WatchChips.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/agent-shortcuts.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/ai-entry-points.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/agent-shortcuts.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/ai-entry-points.test.ts": 3, "apps/webapp/app/components/dashboard-agent/ask-ai-channels.test.ts": 5, - "apps/webapp/app/components/dashboard-agent/askAiOpenRequest.test.ts": 3, - "apps/webapp/app/components/dashboard-agent/chat-layout.test.ts": 7, + "apps/webapp/app/components/dashboard-agent/askAiOpenRequest.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/chat-layout.test.ts": 6, "apps/webapp/app/components/dashboard-agent/coalesced-reload.test.ts": 7, - "apps/webapp/app/components/dashboard-agent/composer-escape.test.ts": 1, - "apps/webapp/app/components/dashboard-agent/dashboardAgentOpenRequest.test.ts": 4, - "apps/webapp/app/components/dashboard-agent/demo/demo.test.ts": 21, + "apps/webapp/app/components/dashboard-agent/composer-escape.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/dashboardAgentOpenRequest.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/demo/demo.test.ts": 18, "apps/webapp/app/components/dashboard-agent/diagnosis-actions.test.ts": 3, "apps/webapp/app/components/dashboard-agent/explicit-prompt.test.ts": 3, - "apps/webapp/app/components/dashboard-agent/header-labels.test.ts": 1, + "apps/webapp/app/components/dashboard-agent/header-labels.test.ts": 2, "apps/webapp/app/components/dashboard-agent/investigate-prompts.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/investigation-winners.test.ts": 9, - "apps/webapp/app/components/dashboard-agent/last-chat-storage.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/message-limits.test.ts": 3, - "apps/webapp/app/components/dashboard-agent/message-order.test.ts": 4, - "apps/webapp/app/components/dashboard-agent/message-quota.test.ts": 6, - "apps/webapp/app/components/dashboard-agent/model-markdown.test.ts": 142, - "apps/webapp/app/components/dashboard-agent/navigate-target.test.ts": 7, - "apps/webapp/app/components/dashboard-agent/opened-chat.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/investigation-winners.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/last-chat-storage.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/message-limits.test.ts": 30, + "apps/webapp/app/components/dashboard-agent/message-order.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/message-quota.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/model-markdown.test.ts": 91, + "apps/webapp/app/components/dashboard-agent/navigate-target.test.ts": 9, + "apps/webapp/app/components/dashboard-agent/opened-chat.test.ts": 3, "apps/webapp/app/components/dashboard-agent/page-label.test.ts": 3, - "apps/webapp/app/components/dashboard-agent/panel-escape.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/pending-intents.test.ts": 7, - "apps/webapp/app/components/dashboard-agent/pending-turn.test.ts": 3, - "apps/webapp/app/components/dashboard-agent/progress-line.test.ts": 5, - "apps/webapp/app/components/dashboard-agent/report-block-adapter.test.ts": 5, - "apps/webapp/app/components/dashboard-agent/report-spark.test.ts": 97, - "apps/webapp/app/components/dashboard-agent/resolve-uris.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/panel-escape.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/pending-intents.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/pending-turn.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/progress-line.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/report-block-adapter.test.ts": 12, + "apps/webapp/app/components/dashboard-agent/report-spark.test.ts": 210, + "apps/webapp/app/components/dashboard-agent/resolve-uris.test.ts": 4, "apps/webapp/app/components/dashboard-agent/retry-action.test.ts": 3, "apps/webapp/app/components/dashboard-agent/run-id.test.ts": 3, "apps/webapp/app/components/dashboard-agent/send-request.test.ts": 2, "apps/webapp/app/components/dashboard-agent/settled-transcript.test.ts": 6, - "apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts": 19, - "apps/webapp/app/components/dashboard-agent/suggested-prompts/promoted.test.ts": 3, - "apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts": 26, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts": 25, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/promoted.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts": 47, "apps/webapp/app/components/dashboard-agent/thinking-marker.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/tool-labels.test.ts": 1, - "apps/webapp/app/components/dashboard-agent/tooltip-accessible-name.test.ts": 1, + "apps/webapp/app/components/dashboard-agent/tool-labels.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/tooltip-accessible-name.test.ts": 2, "apps/webapp/app/components/dashboard-agent/turn-error.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/turn-navigation.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/turn-navigation.test.ts": 5, "apps/webapp/app/components/dashboard-agent/turn-teardown.test.ts": 3, - "apps/webapp/app/components/dashboard-agent/unread-counts.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/unread-counts.test.ts": 5, "apps/webapp/app/components/dashboard-agent/unread-work.test.ts": 2, "apps/webapp/app/components/dashboard-agent/view-actions.test.ts": 6, - "apps/webapp/app/components/dashboard-agent/view-blocks.test.ts": 3, - "apps/webapp/app/components/dashboard-agent/view-catalog.test.ts": 7, - "apps/webapp/app/components/dashboard-agent/wake-banner.test.ts": 11, - "apps/webapp/app/components/dashboard-agent/wake-poll.test.ts": 11, - "apps/webapp/app/components/dashboard-agent/watch-activity.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts": 2, - "apps/webapp/app/components/dashboard-agent/watch-card.test.ts": 33, + "apps/webapp/app/components/dashboard-agent/view-blocks.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/view-catalog.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/wake-banner.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/wake-poll.test.ts": 8, + "apps/webapp/app/components/dashboard-agent/watch-activity.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/watch-card.test.ts": 35, "apps/webapp/app/components/dashboard-agent/watch-chips.test.ts": 3, - "apps/webapp/app/components/queues/queue-name.test.ts": 1, - "apps/webapp/app/components/queues/queue-thresholds.test.ts": 2, - "apps/webapp/app/presenters/v3/reports/report-layout.test.ts": 23, - "apps/webapp/app/routes/storybook.agent-ui/fixtures.test.ts": 6, + "apps/webapp/app/components/queues/queue-name.test.ts": 2, + "apps/webapp/app/components/queues/queue-thresholds.test.ts": 3, + "apps/webapp/app/presenters/v3/reports/report-layout.test.ts": 19, + "apps/webapp/app/routes/storybook.agent-ui/fixtures.test.ts": 7, "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 3, - "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 2, - "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 3318, - "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 7208, - "apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts": 2786, - "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 8861, - "apps/webapp/app/utils/apiKeys.test.ts": 6, - "apps/webapp/app/utils/boundedRequestBody.server.test.ts": 9, - "apps/webapp/app/utils/cspImageOrigins.test.ts": 3, - "apps/webapp/app/utils/databaseMetrics.server.test.ts": 1, + "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 3, + "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 49314, + "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 130135, + "apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts": 49166, + "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 131000, + "apps/webapp/app/utils/apiKeys.test.ts": 5, + "apps/webapp/app/utils/boundedRequestBody.server.test.ts": 21, + "apps/webapp/app/utils/cspImageOrigins.test.ts": 5, + "apps/webapp/app/utils/databaseMetrics.server.test.ts": 3, "apps/webapp/app/utils/deeplinkPages.test.ts": 12, - "apps/webapp/app/utils/environmentAccess.test.ts": 2, - "apps/webapp/app/utils/friendlyId.test.ts": 6, - "apps/webapp/app/utils/impersonationPaths.test.ts": 5, + "apps/webapp/app/utils/environmentAccess.test.ts": 3, + "apps/webapp/app/utils/friendlyId.test.ts": 4, + "apps/webapp/app/utils/impersonationPaths.test.ts": 4, "apps/webapp/app/utils/impersonationState.test.ts": 2, - "apps/webapp/app/utils/localHostGuard.test.ts": 2, - "apps/webapp/app/utils/logSearch.test.ts": 5, - "apps/webapp/app/utils/nullBytes.test.ts": 1, - "apps/webapp/app/utils/pageSwitching.test.ts": 19, - "apps/webapp/app/utils/pageTitle.test.ts": 4, - "apps/webapp/app/utils/plainCustomerCards.test.ts": 6, - "apps/webapp/app/utils/prismaConnectionUrl.test.ts": 1, - "apps/webapp/app/utils/requestIdempotency.test.ts": 1, + "apps/webapp/app/utils/localHostGuard.test.ts": 3, + "apps/webapp/app/utils/logSearch.test.ts": 4, + "apps/webapp/app/utils/nullBytes.test.ts": 3, + "apps/webapp/app/utils/pageSwitching.test.ts": 35, + "apps/webapp/app/utils/pageTitle.test.ts": 5, + "apps/webapp/app/utils/plainCustomerCards.test.ts": 5, + "apps/webapp/app/utils/prismaConnectionUrl.test.ts": 2, + "apps/webapp/app/utils/requestIdempotency.test.ts": 3, "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 4, - "apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts": 6, - "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.dispatchFreshness.test.ts": 6423, - "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 4780, - "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 2, + "apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts": 5, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.dispatchFreshness.test.ts": 4502, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 2898, + "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 5, "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 5, - "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 13, - "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 8268, - "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 1, - "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 10, - "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 5, - "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 8273, - "apps/webapp/app/v3/runStore.server.test.ts": 8656, - "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 22710, - "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 3, - "apps/webapp/app/v3/utils/priority.test.ts": 2, - "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 5039, - "apps/webapp/test/GCRARateLimiter.test.ts": 4477, - "apps/webapp/test/SpanPresenter.readthrough.test.ts": 9160, - "apps/webapp/test/activitySeries.server.test.ts": 6, + "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 7, + "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 7072, + "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 2, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 4, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 6, + "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 5978, + "apps/webapp/app/v3/runStore.server.test.ts": 9961, + "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 5609, + "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 4, + "apps/webapp/app/v3/utils/priority.test.ts": 3, + "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 2984, + "apps/webapp/test/GCRARateLimiter.test.ts": 4553, + "apps/webapp/test/SpanPresenter.readthrough.test.ts": 7526, + "apps/webapp/test/activitySeries.server.test.ts": 4, "apps/webapp/test/additionalApiKeyIssuance.test.ts": 3, - "apps/webapp/test/aiTitleRateLimiter.test.ts": 55, + "apps/webapp/test/aiTitleRateLimiter.test.ts": 165, "apps/webapp/test/api-auth.e2e.test.ts": 20090, - "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 30560, - "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 17241, + "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 93816, + "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 174685, "apps/webapp/test/apiAuthActorClaim.test.ts": 4, - "apps/webapp/test/apiAuthScope.test.ts": 7, - "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 16921, - "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 7297, - "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 8909, - "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 1130, - "apps/webapp/test/apiBuilderAuthorization.test.ts": 1, - "apps/webapp/test/apiKeysPresenter.test.ts": 10287, - "apps/webapp/test/apiRateLimitJwtActor.test.ts": 4, - "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 946, - "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 8834, - "apps/webapp/test/apiRunListPresenter.readthrough.test.ts": 31646, - "apps/webapp/test/apiRunListPresenter.test.ts": 5271, - "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 9048, - "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 1713, - "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 11841, + "apps/webapp/test/apiAuthScope.test.ts": 21, + "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 5924, + "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 5100, + "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 7275, + "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 3030, + "apps/webapp/test/apiBuilderAuthorization.test.ts": 2, + "apps/webapp/test/apiKeysPresenter.test.ts": 9299, + "apps/webapp/test/apiRateLimitJwtActor.test.ts": 6, + "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 2556, + "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 7084, + "apps/webapp/test/apiRunListPresenter.readthrough.test.ts": 23000, + "apps/webapp/test/apiRunListPresenter.test.ts": 93935, + "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 7573, + "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 3367, + "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 10909, "apps/webapp/test/authFeatureControls.test.ts": 3, - "apps/webapp/test/authorizationCodeConsent.test.ts": 10286, + "apps/webapp/test/authorizationCodeConsent.test.ts": 9295, "apps/webapp/test/authorizationRateLimitMiddleware.test.ts": 1, - "apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts": 107, - "apps/webapp/test/batchListPresenter.readroute.test.ts": 12433, - "apps/webapp/test/batchPresenter.test.ts": 15117, - "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 6, - "apps/webapp/test/batchRunAccess.test.ts": 9831, - "apps/webapp/test/batchServices.replicaLag.test.ts": 20440, - "apps/webapp/test/batchStreamGrants.test.ts": 199, - "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 8523, - "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 5, - "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 8063, + "apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts": 221, + "apps/webapp/test/batchListPresenter.readroute.test.ts": 9942, + "apps/webapp/test/batchPresenter.test.ts": 12839, + "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 8, + "apps/webapp/test/batchRunAccess.test.ts": 8683, + "apps/webapp/test/batchServices.replicaLag.test.ts": 4174, + "apps/webapp/test/batchStreamGrants.test.ts": 410, + "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 6059, + "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 3, + "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 7158, "apps/webapp/test/billingAlertsDefaults.test.ts": 2, - "apps/webapp/test/billingAlertsFormat.test.ts": 2, - "apps/webapp/test/billingLimit.schemas.test.ts": 7, - "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 16050, - "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 6177, - "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 2, - "apps/webapp/test/billingLimitConvergeResolve.test.ts": 15, - "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 5, + "apps/webapp/test/billingAlertsFormat.test.ts": 11, + "apps/webapp/test/billingLimit.schemas.test.ts": 6, + "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 13499, + "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 3140, + "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 3, + "apps/webapp/test/billingLimitConvergeResolve.test.ts": 8, + "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 4, "apps/webapp/test/billingLimitHit.test.ts": 3, "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 2, - "apps/webapp/test/billingLimitQueuedRuns.test.ts": 18328, - "apps/webapp/test/billingLimitReconcileTick.test.ts": 7, - "apps/webapp/test/billingLimitReconciliation.test.ts": 13124, + "apps/webapp/test/billingLimitQueuedRuns.test.ts": 18429, + "apps/webapp/test/billingLimitReconcileTick.test.ts": 6, + "apps/webapp/test/billingLimitReconciliation.test.ts": 3178, "apps/webapp/test/billingLimitResolve.test.ts": 2, "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 2, - "apps/webapp/test/billingLimitsRoute.test.ts": 16, + "apps/webapp/test/billingLimitsRoute.test.ts": 21, "apps/webapp/test/branchableEnvironment.test.ts": 3, - "apps/webapp/test/bufferedTriggerPayload.test.ts": 4, - "apps/webapp/test/bulkActionV2.replicaLag.test.ts": 8967, - "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 8014, - "apps/webapp/test/calculateNextSchedule.test.ts": 208, - "apps/webapp/test/cancelRouteReplicaLag.guard.test.ts": 9736, - "apps/webapp/test/chartActivityTimeAxis.test.ts": 21, - "apps/webapp/test/chartXAxisTicks.test.ts": 7, - "apps/webapp/test/chartZoomRange.test.ts": 4, - "apps/webapp/test/chat-snapshot-integration.test.ts": 764, - "apps/webapp/test/checkPermissions.test.ts": 2, - "apps/webapp/test/checkSchedule.test.ts": 10866, + "apps/webapp/test/bufferedTriggerPayload.test.ts": 3, + "apps/webapp/test/bulkActionV2.replicaLag.test.ts": 3510, + "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 6198, + "apps/webapp/test/calculateNextSchedule.test.ts": 177, + "apps/webapp/test/cancelRouteReplicaLag.guard.test.ts": 3023, + "apps/webapp/test/cancelSupersededDeployments.test.ts": 8, + "apps/webapp/test/chartActivityTimeAxis.test.ts": 12, + "apps/webapp/test/chartXAxisTicks.test.ts": 9, + "apps/webapp/test/chartZoomRange.test.ts": 2, + "apps/webapp/test/chat-snapshot-integration.test.ts": 3333, + "apps/webapp/test/checkPermissions.test.ts": 3, + "apps/webapp/test/checkSchedule.test.ts": 9949, "apps/webapp/test/claimTtl.test.ts": 2, - "apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts": 608, - "apps/webapp/test/clickhouseFactory.test.ts": 7018, - "apps/webapp/test/components/DateTime.test.ts": 18, - "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 7, - "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 84, + "apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts": 6102, + "apps/webapp/test/clickhouseFactory.test.ts": 3934, + "apps/webapp/test/components/DateTime.test.ts": 13, + "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 10, + "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 109, "apps/webapp/test/components/runs/v3/RunTag.test.ts": 4, - "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 2, + "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 3, "apps/webapp/test/components/webhookDeliveries/buildDeliveryTimelineItems.test.ts": 4, - "apps/webapp/test/computeBucket.test.ts": 97, - "apps/webapp/test/computeMigration.test.ts": 2, - "apps/webapp/test/concurrencySystemPercentOverride.test.ts": 10891, - "apps/webapp/test/concurrentFlushScheduler.test.ts": 369, - "apps/webapp/test/contextlessPatRoutes.test.ts": 29, - "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 10196, - "apps/webapp/test/createEnvironmentApiKey.test.ts": 11666, - "apps/webapp/test/crossSeamGuard.proof.test.ts": 9872, - "apps/webapp/test/dashboardAgentAlertAdminPreview.test.ts": 2, - "apps/webapp/test/dashboardAgentBodyCap.test.ts": 83, - "apps/webapp/test/dashboardAgentChatRetention.test.ts": 1550, - "apps/webapp/test/dashboardAgentClientMetadata.test.ts": 15, - "apps/webapp/test/dashboardAgentCreateChatOrdering.test.ts": 11, - "apps/webapp/test/dashboardAgentDurableResume.test.ts": 5441, - "apps/webapp/test/dashboardAgentEvalPolicyAuth.test.ts": 75, - "apps/webapp/test/dashboardAgentForeignChat.test.ts": 11, - "apps/webapp/test/dashboardAgentHeadStart.test.ts": 3, - "apps/webapp/test/dashboardAgentImageCsp.test.ts": 2, - "apps/webapp/test/dashboardAgentInProxyMintFailure.test.ts": 6, - "apps/webapp/test/dashboardAgentInvestigationSettlementCard.test.ts": 5, + "apps/webapp/test/computeBucket.test.ts": 96, + "apps/webapp/test/computeMigration.test.ts": 4, + "apps/webapp/test/concurrencySystemPercentOverride.test.ts": 4088, + "apps/webapp/test/concurrentFlushScheduler.test.ts": 356, + "apps/webapp/test/contextlessPatRoutes.test.ts": 25, + "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 9165, + "apps/webapp/test/createEnvironmentApiKey.test.ts": 11337, + "apps/webapp/test/crossSeamGuard.proof.test.ts": 5184, + "apps/webapp/test/dashboardAgentAlertAdminPreview.test.ts": 3, + "apps/webapp/test/dashboardAgentBodyCap.test.ts": 106, + "apps/webapp/test/dashboardAgentChatRetention.test.ts": 3030, + "apps/webapp/test/dashboardAgentClientMetadata.test.ts": 18, + "apps/webapp/test/dashboardAgentCreateChatOrdering.test.ts": 12, + "apps/webapp/test/dashboardAgentDurableResume.test.ts": 4269, + "apps/webapp/test/dashboardAgentEvalPolicyAuth.test.ts": 47, + "apps/webapp/test/dashboardAgentForeignChat.test.ts": 6, + "apps/webapp/test/dashboardAgentHeadStart.test.ts": 5, + "apps/webapp/test/dashboardAgentImageCsp.test.ts": 3, + "apps/webapp/test/dashboardAgentInProxyMintFailure.test.ts": 9, + "apps/webapp/test/dashboardAgentInvestigationSettlementCard.test.ts": 3, "apps/webapp/test/dashboardAgentInvestigationWinner.test.ts": 2, - "apps/webapp/test/dashboardAgentLastReadBackfill.test.ts": 11790, - "apps/webapp/test/dashboardAgentLegacyMessagesColumn.test.ts": 396, - "apps/webapp/test/dashboardAgentMessageCards.test.ts": 12, - "apps/webapp/test/dashboardAgentMessageSurrogate.test.ts": 2556, - "apps/webapp/test/dashboardAgentQueriesTenantIsolation.test.ts": 5634, - "apps/webapp/test/dashboardAgentQuota.test.ts": 17173, - "apps/webapp/test/dashboardAgentRoutes.test.ts": 22, - "apps/webapp/test/dashboardAgentSurrogatePersist.test.ts": 6426, - "apps/webapp/test/dashboardAgentTenantIsolation.test.ts": 11316, - "apps/webapp/test/dashboardAgentToolScopes.test.ts": 1, - "apps/webapp/test/dashboardAgentTranscriptStore.test.ts": 11851, - "apps/webapp/test/dashboardAgentUnreadWorkScope.test.ts": 1730, - "apps/webapp/test/dashboardAgentWakeActivity.test.ts": 2441, - "apps/webapp/test/dashboardAgentWatchAlertEnvironments.test.ts": 3514, - "apps/webapp/test/dashboardAgentWatchAlertFanout.test.ts": 16, - "apps/webapp/test/dashboardAgentWatchAlertGate.test.ts": 2, - "apps/webapp/test/dashboardAgentWatchAlertOwnerScope.test.ts": 1725, - "apps/webapp/test/dashboardAgentWatchBatchFairness.test.ts": 3909, - "apps/webapp/test/dashboardAgentWatchBatchRecording.test.ts": 8074, - "apps/webapp/test/dashboardAgentWatchCardAtomicity.test.ts": 7569, - "apps/webapp/test/dashboardAgentWatchCardRequestId.test.ts": 880, - "apps/webapp/test/dashboardAgentWatchChecks.test.ts": 7, - "apps/webapp/test/dashboardAgentWatchCreationReads.test.ts": 2, - "apps/webapp/test/dashboardAgentWatchErrorFingerprint.test.ts": 4824, - "apps/webapp/test/dashboardAgentWatchInvestigate.test.ts": 18, - "apps/webapp/test/dashboardAgentWatchLimitStatus.test.ts": 1830, - "apps/webapp/test/dashboardAgentWatchLimits.test.ts": 14463, - "apps/webapp/test/dashboardAgentWatchQueueAge.test.ts": 2, - "apps/webapp/test/dashboardAgentWatchQueueName.test.ts": 14784, - "apps/webapp/test/dashboardAgentWatchSweepAlertOnce.test.ts": 1237, - "apps/webapp/test/dashboardAgentWatchSweepBoundary.test.ts": 8660, - "apps/webapp/test/dashboardAgentWatchTenancy.test.ts": 5475, - "apps/webapp/test/dashboardAgentWatchToken.test.ts": 16, - "apps/webapp/test/dashboardAgentWatchWording.test.ts": 7, - "apps/webapp/test/dashboardAgentWatches.batch.test.ts": 34184, - "apps/webapp/test/dashboardAgentWatches.delivery.test.ts": 81151, - "apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts": 80861, - "apps/webapp/test/deleteTaskSchedule.test.ts": 16973, - "apps/webapp/test/deliveryIdBounds.test.ts": 18, - "apps/webapp/test/dependentAttemptScope.test.ts": 2, + "apps/webapp/test/dashboardAgentLastReadBackfill.test.ts": 3079, + "apps/webapp/test/dashboardAgentLegacyMessagesColumn.test.ts": 123, + "apps/webapp/test/dashboardAgentMessageCards.test.ts": 9, + "apps/webapp/test/dashboardAgentMessageSurrogate.test.ts": 3020, + "apps/webapp/test/dashboardAgentQueriesTenantIsolation.test.ts": 3954, + "apps/webapp/test/dashboardAgentQuota.test.ts": 3891, + "apps/webapp/test/dashboardAgentRoutes.test.ts": 10, + "apps/webapp/test/dashboardAgentSurrogatePersist.test.ts": 3951, + "apps/webapp/test/dashboardAgentTenantIsolation.test.ts": 4250, + "apps/webapp/test/dashboardAgentToolScopes.test.ts": 4, + "apps/webapp/test/dashboardAgentTranscriptStore.test.ts": 8600, + "apps/webapp/test/dashboardAgentUnreadWorkScope.test.ts": 3104, + "apps/webapp/test/dashboardAgentWakeActivity.test.ts": 3041, + "apps/webapp/test/dashboardAgentWatchAlertEnvironments.test.ts": 3993, + "apps/webapp/test/dashboardAgentWatchAlertFanout.test.ts": 9, + "apps/webapp/test/dashboardAgentWatchAlertGate.test.ts": 3, + "apps/webapp/test/dashboardAgentWatchAlertOwnerScope.test.ts": 3187, + "apps/webapp/test/dashboardAgentWatchBatchFairness.test.ts": 4002, + "apps/webapp/test/dashboardAgentWatchBatchRecording.test.ts": 4161, + "apps/webapp/test/dashboardAgentWatchCardAtomicity.test.ts": 4305, + "apps/webapp/test/dashboardAgentWatchCardRequestId.test.ts": 3168, + "apps/webapp/test/dashboardAgentWatchChecks.test.ts": 16, + "apps/webapp/test/dashboardAgentWatchCreationReads.test.ts": 3, + "apps/webapp/test/dashboardAgentWatchErrorFingerprint.test.ts": 3026, + "apps/webapp/test/dashboardAgentWatchInvestigate.test.ts": 16, + "apps/webapp/test/dashboardAgentWatchLimitStatus.test.ts": 2800, + "apps/webapp/test/dashboardAgentWatchLimits.test.ts": 5568, + "apps/webapp/test/dashboardAgentWatchQueueAge.test.ts": 3, + "apps/webapp/test/dashboardAgentWatchQueueName.test.ts": 4369, + "apps/webapp/test/dashboardAgentWatchSweepAlertOnce.test.ts": 2843, + "apps/webapp/test/dashboardAgentWatchSweepBoundary.test.ts": 3481, + "apps/webapp/test/dashboardAgentWatchTenancy.test.ts": 4237, + "apps/webapp/test/dashboardAgentWatchToken.test.ts": 22, + "apps/webapp/test/dashboardAgentWatchWording.test.ts": 5, + "apps/webapp/test/dashboardAgentWatches.batch.test.ts": 93476, + "apps/webapp/test/dashboardAgentWatches.delivery.test.ts": 17111, + "apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts": 16072, + "apps/webapp/test/deleteTaskSchedule.test.ts": 7573, + "apps/webapp/test/deliveryIdBounds.test.ts": 19, + "apps/webapp/test/dependentAttemptScope.test.ts": 4, "apps/webapp/test/deploymentApiPaths.test.ts": 2, - "apps/webapp/test/detectQueryTables.test.ts": 97, - "apps/webapp/test/detectbadJsonStrings.test.ts": 81, - "apps/webapp/test/devBranchServices.test.ts": 3555, - "apps/webapp/test/devPresenceRecency.test.ts": 168, - "apps/webapp/test/directorySyncEffects.server.test.ts": 7, - "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2465, - "apps/webapp/test/duplicateTaskIds.test.ts": 3, - "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1532, - "apps/webapp/test/emailPattern.test.ts": 4, - "apps/webapp/test/engine/batchPayloads.test.ts": 5017, - "apps/webapp/test/engine/dequeueWorkerVersionFreshness.test.ts": 10262, - "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 11035, - "apps/webapp/test/engine/streamBatchItems.test.ts": 22941, - "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 7090, - "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 17525, - "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 6851, - "apps/webapp/test/engine/triggerTask.debounce.test.ts": 12067, - "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 11663, - "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 7667, - "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 7963, - "apps/webapp/test/engine/triggerTask.residency.test.ts": 7611, - "apps/webapp/test/engine/triggerTask.test.ts": 10375, - "apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts": 8985, - "apps/webapp/test/env.server.test.ts": 500, - "apps/webapp/test/envConcurrencyLimitPause.server.test.ts": 4829, - "apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts": 4856, - "apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts": 3077, - "apps/webapp/test/envJwtActorClaim.test.ts": 20, - "apps/webapp/test/envParamRoute.ownership.test.ts": 5, - "apps/webapp/test/environmentSort.test.ts": 8, - "apps/webapp/test/environmentVariableApiAccess.test.ts": 7, + "apps/webapp/test/detectQueryTables.test.ts": 159, + "apps/webapp/test/detectbadJsonStrings.test.ts": 58, + "apps/webapp/test/devBranchServices.test.ts": 4992, + "apps/webapp/test/devPresenceRecency.test.ts": 209, + "apps/webapp/test/directorySyncEffects.server.test.ts": 12, + "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2555, + "apps/webapp/test/duplicateTaskIds.test.ts": 2, + "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1623, + "apps/webapp/test/emailPattern.test.ts": 5, + "apps/webapp/test/engine/batchPayloads.test.ts": 5016, + "apps/webapp/test/engine/dequeueWorkerVersionFreshness.test.ts": 49054, + "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 48923, + "apps/webapp/test/engine/streamBatchItems.test.ts": 22321, + "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 4029, + "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 131643, + "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 90520, + "apps/webapp/test/engine/triggerTask.debounce.test.ts": 213034, + "apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts": 125439, + "apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts": 125439, + "apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts": 125439, + "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 171570, + "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 172152, + "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 171782, + "apps/webapp/test/engine/triggerTask.residency.test.ts": 171372, + "apps/webapp/test/engine/triggerTask.test.ts": 171950, + "apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts": 3608, + "apps/webapp/test/env.server.test.ts": 573, + "apps/webapp/test/envConcurrencyLimitPause.server.test.ts": 8769, + "apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts": 9014, + "apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts": 7489, + "apps/webapp/test/envJwtActorClaim.test.ts": 24, + "apps/webapp/test/envParamRoute.ownership.test.ts": 4, + "apps/webapp/test/environmentSort.test.ts": 9, + "apps/webapp/test/environmentVariableApiAccess.test.ts": 8, "apps/webapp/test/environmentVariableDeduplication.test.ts": 3, - "apps/webapp/test/environmentVariableRules.test.ts": 2, - "apps/webapp/test/environmentVariablesEnvironments.test.ts": 5637, - "apps/webapp/test/environmentVariablesReplicaRouting.test.ts": 9602, - "apps/webapp/test/environmentVariablesRepository.test.ts": 4139, - "apps/webapp/test/errorFingerprinting.test.ts": 5, - "apps/webapp/test/errorGroupWebhook.test.ts": 10, - "apps/webapp/test/featureFlags.test.ts": 7936, - "apps/webapp/test/findEnvironmentByApiKey.test.ts": 10783, - "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 15826, - "apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts": 6053, - "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 9970, - "apps/webapp/test/getDeploymentImageRef.test.ts": 6, - "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 466, - "apps/webapp/test/googleEmailVerification.test.ts": 2, + "apps/webapp/test/environmentVariableRules.test.ts": 3, + "apps/webapp/test/environmentVariablesEnvironments.test.ts": 3912, + "apps/webapp/test/environmentVariablesReplicaRouting.test.ts": 8817, + "apps/webapp/test/environmentVariablesRepository.test.ts": 4096, + "apps/webapp/test/errorFingerprinting.test.ts": 9, + "apps/webapp/test/errorGroupWebhook.test.ts": 6, + "apps/webapp/test/externalDeploymentCache.test.ts": 160, + "apps/webapp/test/featureFlags.test.ts": 4158, + "apps/webapp/test/findEnvironmentByApiKey.test.ts": 5169, + "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 5425, + "apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts": 3375, + "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 10112, + "apps/webapp/test/getDeploymentImageRef.test.ts": 7, + "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 6140, + "apps/webapp/test/googleEmailVerification.test.ts": 4, "apps/webapp/test/healthcheck-require-plugins.e2e.test.ts": 37215, - "apps/webapp/test/httpErrors.test.ts": 2, - "apps/webapp/test/idempotencyDedupResidency.test.ts": 7600, - "apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts": 2, - "apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts": 14676, - "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 17731, - "apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts": 14016, - "apps/webapp/test/impersonationConsent.test.ts": 10012, + "apps/webapp/test/httpErrors.test.ts": 18, + "apps/webapp/test/idempotencyDedupResidency.test.ts": 5951, + "apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts": 7, + "apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts": 11744, + "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 5913, + "apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts": 5501, + "apps/webapp/test/impersonationConsent.test.ts": 9467, "apps/webapp/test/internalApiOrigin.test.ts": 2, "apps/webapp/test/inviteRoleLadder.test.ts": 2, - "apps/webapp/test/logger.server.onError.test.ts": 37, - "apps/webapp/test/logsSearchProjector.test.ts": 7, - "apps/webapp/test/logsSearchProjectorRedisStore.test.ts": 36, - "apps/webapp/test/logsSearchProjectorStateStore.test.ts": 729, - "apps/webapp/test/member.server.test.ts": 9006, - "apps/webapp/test/memberDevEnvironments.server.test.ts": 7442, + "apps/webapp/test/logger.server.onError.test.ts": 19, + "apps/webapp/test/logsSearchProjector.test.ts": 9, + "apps/webapp/test/logsSearchProjectorRedisStore.test.ts": 190, + "apps/webapp/test/logsSearchProjectorStateStore.test.ts": 2715, + "apps/webapp/test/member.server.test.ts": 7181, + "apps/webapp/test/memberDevEnvironments.server.test.ts": 5862, "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 5, - "apps/webapp/test/metadataRouteReplicaLag.guard.test.ts": 13, - "apps/webapp/test/mfaRateLimiter.test.ts": 331, - "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 494, - "apps/webapp/test/mollifierClaimResolution.test.ts": 4, - "apps/webapp/test/mollifierDecisionLabels.test.ts": 2, - "apps/webapp/test/mollifierDrainerHandler.test.ts": 19, - "apps/webapp/test/mollifierDrainerWorker.test.ts": 4, + "apps/webapp/test/metadataRouteReplicaLag.guard.test.ts": 7, + "apps/webapp/test/mfaRateLimiter.test.ts": 174, + "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 560, + "apps/webapp/test/mollifierClaimResolution.test.ts": 8, + "apps/webapp/test/mollifierDecisionLabels.test.ts": 5, + "apps/webapp/test/mollifierDrainerHandler.test.ts": 12, + "apps/webapp/test/mollifierDrainerWorker.test.ts": 6, "apps/webapp/test/mollifierDrainingGauge.test.ts": 448, - "apps/webapp/test/mollifierGate.test.ts": 11, - "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 16, + "apps/webapp/test/mollifierGate.test.ts": 14, + "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 10, "apps/webapp/test/mollifierMollify.test.ts": 6, - "apps/webapp/test/mollifierMutateWithFallback.test.ts": 4, - "apps/webapp/test/mollifierReadFallback.test.ts": 11, - "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 3, + "apps/webapp/test/mollifierMutateWithFallback.test.ts": 10, + "apps/webapp/test/mollifierReadFallback.test.ts": 14, + "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 2, "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 8, - "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 6, - "apps/webapp/test/mollifierStaleSweep.test.ts": 1030, - "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 3, - "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 2, - "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 63, - "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 2, - "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 1, - "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 5, - "apps/webapp/test/mollifierSyntheticTrace.test.ts": 4, - "apps/webapp/test/mollifierTripEvaluator.test.ts": 54, - "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 25633, - "apps/webapp/test/objectStore.test.ts": 11175, - "apps/webapp/test/orgBanner.test.ts": 2, - "apps/webapp/test/orgMember.server.test.ts": 3927, - "apps/webapp/test/organizationDataStoresRegistry.test.ts": 31981, - "apps/webapp/test/otlpExporter.test.ts": 6, - "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 146, - "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 937, - "apps/webapp/test/pauseEnvironment.server.test.ts": 10560, - "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 7527, - "apps/webapp/test/platformNotifications.test.ts": 8, - "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 10293, - "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 100, - "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 37569, - "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 3, - "apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts": 14584, - "apps/webapp/test/prismaErrors.test.ts": 1, - "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 4980, - "apps/webapp/test/projectEnvironmentCredentialRoute.test.ts": 5, - "apps/webapp/test/projectEnvironmentsBranchScope.test.ts": 46, - "apps/webapp/test/projectSettingsToastRedirect.test.ts": 12, + "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 5, + "apps/webapp/test/mollifierStaleSweep.test.ts": 482, + "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 4, + "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 8, + "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 196, + "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 4, + "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 3, + "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 6, + "apps/webapp/test/mollifierSyntheticTrace.test.ts": 5, + "apps/webapp/test/mollifierTripEvaluator.test.ts": 155, + "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 24036, + "apps/webapp/test/objectStore.test.ts": 6327, + "apps/webapp/test/orgBanner.test.ts": 3, + "apps/webapp/test/orgMember.server.test.ts": 4296, + "apps/webapp/test/organizationDataStoresRegistry.test.ts": 6996, + "apps/webapp/test/otlpExporter.test.ts": 8, + "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 5680, + "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 81, + "apps/webapp/test/pauseEnvironment.server.test.ts": 9596, + "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 5566, + "apps/webapp/test/platformNotifications.test.ts": 6, + "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 8879, + "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 5563, + "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 31639, + "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 2, + "apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts": 3561, + "apps/webapp/test/prismaErrors.test.ts": 2, + "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 3785, + "apps/webapp/test/projectEnvironmentCredentialRoute.test.ts": 10, + "apps/webapp/test/projectEnvironmentsBranchScope.test.ts": 33, + "apps/webapp/test/projectSettingsToastRedirect.test.ts": 13, "apps/webapp/test/promptOverrideSource.test.ts": 2, - "apps/webapp/test/publicAccessTokenResponse.test.ts": 48, - "apps/webapp/test/publicTokensRoute.test.ts": 1556, - "apps/webapp/test/publishClaimResult.test.ts": 1, - "apps/webapp/test/queryResultsTimeTicks.test.ts": 2, - "apps/webapp/test/queryRouteReadOnly.test.ts": 63, - "apps/webapp/test/queryScope.test.ts": 3, - "apps/webapp/test/queueDepthSeries.test.ts": 6, + "apps/webapp/test/publicAccessTokenResponse.test.ts": 41, + "apps/webapp/test/publicTokensRoute.test.ts": 3774, + "apps/webapp/test/publishClaimResult.test.ts": 5, + "apps/webapp/test/queryResultsTimeTicks.test.ts": 4, + "apps/webapp/test/queryRouteReadOnly.test.ts": 88, + "apps/webapp/test/queryScope.test.ts": 5, + "apps/webapp/test/queueDepthSeries.test.ts": 3, "apps/webapp/test/queueListPagination.test.ts": 2, "apps/webapp/test/queueMetricsMapping.test.ts": 6, - "apps/webapp/test/queueRetrieveJwt.test.ts": 40, - "apps/webapp/test/queueSparklineGrid.test.ts": 6, - "apps/webapp/test/rbacFallbackBranch.test.ts": 8470, - "apps/webapp/test/rbacFallbackSessionFloor.test.ts": 13998, - "apps/webapp/test/reacquireClearedGlobalWinner.test.ts": 3, - "apps/webapp/test/readBodyWithCap.test.ts": 7, - "apps/webapp/test/readRunForEvent.replicaLag.test.ts": 9913, + "apps/webapp/test/queueRetrieveJwt.test.ts": 27, + "apps/webapp/test/queueSparklineGrid.test.ts": 9, + "apps/webapp/test/rbacFallbackBranch.test.ts": 7170, + "apps/webapp/test/rbacFallbackSessionFloor.test.ts": 4551, + "apps/webapp/test/reacquireClearedGlobalWinner.test.ts": 5, + "apps/webapp/test/readBodyWithCap.test.ts": 15, + "apps/webapp/test/readRunForEvent.replicaLag.test.ts": 8719, "apps/webapp/test/realtime/boundedTtlCache.test.ts": 4, - "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 51762, - "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 8, - "apps/webapp/test/realtime/envChangeRouter.test.ts": 959, - "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 3659, - "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 7, - "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 306, - "apps/webapp/test/realtime/replayCursorStore.test.ts": 1443, - "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 376, - "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3003, + "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 42422, + "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 7, + "apps/webapp/test/realtime/envChangeRouter.test.ts": 942, + "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 4057, + "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 6, + "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 287, + "apps/webapp/test/realtime/replayCursorStore.test.ts": 1134, + "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 372, + "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3207, "apps/webapp/test/realtime/runReaderProjection.test.ts": 3, - "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 10077, - "apps/webapp/test/realtime/shadowCompare.test.ts": 7, - "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 7894, + "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 8141, + "apps/webapp/test/realtime/shadowCompare.test.ts": 4, + "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 5721, "apps/webapp/test/realtimeClient.test.ts": 1, - "apps/webapp/test/realtimeServices.replicaLag.test.ts": 9845, - "apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts": 7384, - "apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts": 18951, + "apps/webapp/test/realtimeServices.replicaLag.test.ts": 4498, + "apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts": 3098, + "apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts": 6331, "apps/webapp/test/realtimeStreamsVersion.test.ts": 3, - "apps/webapp/test/redisRealtimeStreams.test.ts": 5306, - "apps/webapp/test/registryConfig.test.ts": 298, - "apps/webapp/test/reloadingRegistry.test.ts": 2, - "apps/webapp/test/removeTeamMember.test.ts": 11245, - "apps/webapp/test/replay-after-crash.test.ts": 989, - "apps/webapp/test/replayRouteReplicaLag.guard.test.ts": 13268, - "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 2602, - "apps/webapp/test/reportCurationTrust.test.ts": 6, - "apps/webapp/test/reportHealth.test.ts": 16, - "apps/webapp/test/reportHealthData.test.ts": 10, - "apps/webapp/test/reportMetricDelta.test.ts": 20, - "apps/webapp/test/reportPresenter.test.ts": 25, - "apps/webapp/test/reportRenderParity.test.ts": 75, - "apps/webapp/test/reportTrust.test.ts": 2, - "apps/webapp/test/reportsApiRoute.test.ts": 12, - "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 14311, - "apps/webapp/test/resolveBatchForRealtime.test.ts": 1, - "apps/webapp/test/resolveOrgIdFromSlugForUser.test.ts": 2355, - "apps/webapp/test/resolveProjectScopedEnvironments.test.ts": 4, - "apps/webapp/test/resolveTriggerUri.test.ts": 6, - "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 7575, - "apps/webapp/test/routeCspImgSrc.test.ts": 34, - "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 7302, - "apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts": 22148, + "apps/webapp/test/redisRealtimeStreams.test.ts": 5391, + "apps/webapp/test/registryConfig.test.ts": 378, + "apps/webapp/test/reloadingRegistry.test.ts": 3, + "apps/webapp/test/removeTeamMember.test.ts": 10573, + "apps/webapp/test/replay-after-crash.test.ts": 4057, + "apps/webapp/test/replayRouteReplicaLag.guard.test.ts": 5527, + "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 3452, + "apps/webapp/test/reportCurationTrust.test.ts": 19, + "apps/webapp/test/reportHealth.test.ts": 23, + "apps/webapp/test/reportHealthData.test.ts": 12, + "apps/webapp/test/reportMetricDelta.test.ts": 12, + "apps/webapp/test/reportPresenter.test.ts": 26, + "apps/webapp/test/reportRenderParity.test.ts": 53, + "apps/webapp/test/reportTrust.test.ts": 4, + "apps/webapp/test/reportsApiRoute.test.ts": 14, + "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 5575, + "apps/webapp/test/resolveBatchForRealtime.test.ts": 3, + "apps/webapp/test/resolveExternalIdReuse.test.ts": 10008, + "apps/webapp/test/resolveOrgIdFromSlugForUser.test.ts": 2918, + "apps/webapp/test/resolveProjectScopedEnvironments.test.ts": 3, + "apps/webapp/test/resolveTriggerUri.test.ts": 8, + "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 6353, + "apps/webapp/test/routeCspImgSrc.test.ts": 69, + "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 6057, + "apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts": 7087, "apps/webapp/test/runCommitAuthorization.test.ts": 8, - "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 13211, - "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 2, - "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 4283, - "apps/webapp/test/runEngineHandlers.test.ts": 16844, - "apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts": 8775, - "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 10, - "apps/webapp/test/runOpsDbTopology.test.ts": 21098, - "apps/webapp/test/runOpsMintCutover.test.ts": 3734, - "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3972, - "apps/webapp/test/runOpsSplitMode.test.ts": 13489, - "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 2, + "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 5175, + "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 7, + "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 4964, + "apps/webapp/test/runEngineHandlers.test.ts": 16338, + "apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts": 8602, + "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 5, + "apps/webapp/test/runOpsDbTopology.test.ts": 6510, + "apps/webapp/test/runOpsMintCutover.test.ts": 3355, + "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3080, + "apps/webapp/test/runOpsSplitMode.test.ts": 4077, + "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 3, "apps/webapp/test/runOpsSplitReadGate.test.ts": 4, - "apps/webapp/test/runPresenterReadRoute.test.ts": 4891, - "apps/webapp/test/runPresenters.replicaLag.test.ts": 25923, - "apps/webapp/test/runTimestamps.test.ts": 2, - "apps/webapp/test/runsBackfiller.test.ts": 10511, + "apps/webapp/test/runPresenterReadRoute.test.ts": 4214, + "apps/webapp/test/runPresenters.replicaLag.test.ts": 4893, + "apps/webapp/test/runTimestamps.test.ts": 4, + "apps/webapp/test/runsBackfiller.test.ts": 9029, "apps/webapp/test/runsReplicationBenchmark.test.ts": 1, - "apps/webapp/test/runsReplicationInstance.test.ts": 20278, + "apps/webapp/test/runsReplicationInstance.test.ts": 19226, "apps/webapp/test/runsReplicationJsonRecoveryBenchmark.test.ts": 1, - "apps/webapp/test/runsReplicationService.part1.test.ts": 30406, - "apps/webapp/test/runsReplicationService.part10.test.ts": 18427, - "apps/webapp/test/runsReplicationService.part2.test.ts": 26828, - "apps/webapp/test/runsReplicationService.part3.test.ts": 15207, - "apps/webapp/test/runsReplicationService.part4.test.ts": 41894, - "apps/webapp/test/runsReplicationService.part5.test.ts": 10525, - "apps/webapp/test/runsReplicationService.part6.test.ts": 15181, - "apps/webapp/test/runsReplicationService.part7.test.ts": 71460, - "apps/webapp/test/runsReplicationService.part8.test.ts": 30321, - "apps/webapp/test/runsReplicationService.part9.test.ts": 15987, - "apps/webapp/test/runsRepository.part1.test.ts": 28851, - "apps/webapp/test/runsRepository.part2.test.ts": 28205, - "apps/webapp/test/runsRepository.part3.test.ts": 23041, - "apps/webapp/test/runsRepository.part4.test.ts": 25019, - "apps/webapp/test/runsRepository.readthrough.test.ts": 34731, - "apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts": 5349, - "apps/webapp/test/runsRepositoryCpres.test.ts": 7188, - "apps/webapp/test/runsRepositoryCursor.test.ts": 33200, - "apps/webapp/test/safeEnvironmentLog.test.ts": 1, - "apps/webapp/test/safeIntegrationLog.test.ts": 2, - "apps/webapp/test/safeRequestLogContext.test.ts": 4, - "apps/webapp/test/safeWebhookFetch.test.ts": 5, + "apps/webapp/test/runsReplicationService.part1.test.ts": 29667, + "apps/webapp/test/runsReplicationService.part10.test.ts": 15799, + "apps/webapp/test/runsReplicationService.part2.test.ts": 24084, + "apps/webapp/test/runsReplicationService.part3.test.ts": 12625, + "apps/webapp/test/runsReplicationService.part4.test.ts": 31142, + "apps/webapp/test/runsReplicationService.part5.test.ts": 8925, + "apps/webapp/test/runsReplicationService.part6.test.ts": 13400, + "apps/webapp/test/runsReplicationService.part7.test.ts": 69637, + "apps/webapp/test/runsReplicationService.part8.test.ts": 28406, + "apps/webapp/test/runsReplicationService.part9.test.ts": 9244, + "apps/webapp/test/runsReplicationServiceExternalDeploymentId.test.ts": 8023, + "apps/webapp/test/runsRepository.part1.test.ts": 27264, + "apps/webapp/test/runsRepository.part2.test.ts": 23743, + "apps/webapp/test/runsRepository.part3.test.ts": 20003, + "apps/webapp/test/runsRepository.part4.test.ts": 24817, + "apps/webapp/test/runsRepository.readthrough.test.ts": 36287, + "apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts": 3779, + "apps/webapp/test/runsRepositoryCpres.test.ts": 7073, + "apps/webapp/test/runsRepositoryCursor.test.ts": 26167, + "apps/webapp/test/safeEnvironmentLog.test.ts": 2, + "apps/webapp/test/safeIntegrationLog.test.ts": 3, + "apps/webapp/test/safeRequestLogContext.test.ts": 16, + "apps/webapp/test/safeWebhookFetch.test.ts": 4, "apps/webapp/test/safeWebhookUrl.test.ts": 6, - "apps/webapp/test/sameOriginNavigation.test.ts": 2, - "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 9, + "apps/webapp/test/sameOriginNavigation.test.ts": 14, + "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 11, "apps/webapp/test/sanitizeSessionInput.server.test.ts": 3, "apps/webapp/test/sanitizeUrl.test.ts": 2, - "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 4, - "apps/webapp/test/scheduleTimings.test.ts": 1506, - "apps/webapp/test/scheduleWindow.test.ts": 31, - "apps/webapp/test/schedulesPutEnvScoping.test.ts": 11109, - "apps/webapp/test/selectBestEnvironment.test.ts": 1, - "apps/webapp/test/sentryRequestIsolation.test.ts": 65, - "apps/webapp/test/sentryTenantContext.test.ts": 4, - "apps/webapp/test/sentryTraceContext.server.test.ts": 9, - "apps/webapp/test/services.controlPlane.readthrough.test.ts": 7217, - "apps/webapp/test/services/organizationAccessToken.test.ts": 6, - "apps/webapp/test/services/personalAccessToken.test.ts": 7, + "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 3, + "apps/webapp/test/scheduleTimings.test.ts": 2150, + "apps/webapp/test/scheduleWindow.test.ts": 27, + "apps/webapp/test/schedulesPutEnvScoping.test.ts": 10070, + "apps/webapp/test/selectBestEnvironment.test.ts": 4, + "apps/webapp/test/sentryRequestIsolation.test.ts": 73, + "apps/webapp/test/sentryTenantContext.test.ts": 3, + "apps/webapp/test/sentryTraceContext.server.test.ts": 6, + "apps/webapp/test/services.controlPlane.readthrough.test.ts": 4862, + "apps/webapp/test/services/organizationAccessToken.test.ts": 5, + "apps/webapp/test/services/personalAccessToken.test.ts": 6, "apps/webapp/test/session-agent.e2e.test.ts": 73920, "apps/webapp/test/session-stream.browser.e2e.test.ts": 20470, "apps/webapp/test/session-stream.e2e.test.ts": 30404, - "apps/webapp/test/sessionDuration.test.ts": 10855, + "apps/webapp/test/sessionDuration.test.ts": 11876, "apps/webapp/test/sessionRunStreamsBackend.e2e.test.ts": 20771, "apps/webapp/test/sessionRunStreamsPerOrgBasin.e2e.test.ts": 20639, - "apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts": 13790, - "apps/webapp/test/sessions.readthrough.test.ts": 9480, - "apps/webapp/test/sessionsReplicationService.test.ts": 19666, - "apps/webapp/test/setActiveOnTaskSchedule.test.ts": 9617, - "apps/webapp/test/shouldRevalidateRunsList.test.ts": 2, + "apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts": 3529, + "apps/webapp/test/sessions.readthrough.test.ts": 6749, + "apps/webapp/test/sessionsReplicationService.test.ts": 17485, + "apps/webapp/test/setActiveOnTaskSchedule.test.ts": 9330, + "apps/webapp/test/shouldRevalidateRunsList.test.ts": 4, "apps/webapp/test/slackErrorAlerts.test.ts": 1, - "apps/webapp/test/slackOAuthResultLog.test.ts": 1, - "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 11747, - "apps/webapp/test/spanTraceRoutes.replicaLag.test.ts": 15558, - "apps/webapp/test/streamBatchItemsAuthorization.test.ts": 4, - "apps/webapp/test/streamLoader.controlPlane.test.ts": 14703, - "apps/webapp/test/syncDeclarativeSchedules.test.ts": 10518, - "apps/webapp/test/syncDeclarativeWebhooks.test.ts": 17825, - "apps/webapp/test/taskCodeSnippets.test.ts": 5, - "apps/webapp/test/tenantContext.test.ts": 27, - "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 3, - "apps/webapp/test/tenantContextResolver.test.ts": 16, + "apps/webapp/test/slackOAuthResultLog.test.ts": 2, + "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 5779, + "apps/webapp/test/spanTraceRoutes.replicaLag.test.ts": 5917, + "apps/webapp/test/streamBatchItemsAuthorization.test.ts": 5, + "apps/webapp/test/streamLoader.controlPlane.test.ts": 4970, + "apps/webapp/test/syncDeclarativeSchedules.test.ts": 9761, + "apps/webapp/test/syncDeclarativeWebhooks.test.ts": 10183, + "apps/webapp/test/taskCodeSnippets.test.ts": 4, + "apps/webapp/test/tenantContext.test.ts": 24, + "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 2, + "apps/webapp/test/tenantContextResolver.test.ts": 15, "apps/webapp/test/themePreference.test.ts": 4, - "apps/webapp/test/timeGranularity.test.ts": 3, - "apps/webapp/test/timelineSpanEvents.test.ts": 3, - "apps/webapp/test/traceExport.test.ts": 7, - "apps/webapp/test/uatEnvironmentClaim.test.ts": 22, - "apps/webapp/test/updateMetadata.test.ts": 27671, - "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 8127, - "apps/webapp/test/useTableSort.test.ts": 9, - "apps/webapp/test/userActorEnvironmentScopeRouteBuilder.test.ts": 42, - "apps/webapp/test/userActorPatOnlyBoundary.test.ts": 4565, - "apps/webapp/test/userActorProjectWideScope.test.ts": 9578, - "apps/webapp/test/userActorSourcePat.test.ts": 85, - "apps/webapp/test/userActorTokenClaimsAndScopes.test.ts": 3360, - "apps/webapp/test/utils/timezones.test.ts": 27, - "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 8003, - "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 8169, - "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 8186, - "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 17785, - "apps/webapp/test/validateGitBranchName.test.ts": 5, - "apps/webapp/test/vercelUrls.test.ts": 4, - "apps/webapp/test/verifyDeploymentImage.test.ts": 621, - "apps/webapp/test/viewAsUser.test.ts": 5, - "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 8169, - "apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts": 7980, - "apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts": 17659, - "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 10934, - "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 16964, - "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 17630, - "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 15593, - "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 17406, - "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 31202, - "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 9618, - "apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts": 8877, - "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 12113, + "apps/webapp/test/timeGranularity.test.ts": 4, + "apps/webapp/test/timelineSpanEvents.test.ts": 6, + "apps/webapp/test/traceExport.test.ts": 6, + "apps/webapp/test/uatEnvironmentClaim.test.ts": 32, + "apps/webapp/test/updateMetadata.test.ts": 20384, + "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 7987, + "apps/webapp/test/useTableSort.test.ts": 7, + "apps/webapp/test/userActorEnvironmentScopeRouteBuilder.test.ts": 28, + "apps/webapp/test/userActorPatOnlyBoundary.test.ts": 3254, + "apps/webapp/test/userActorProjectWideScope.test.ts": 3735, + "apps/webapp/test/userActorSourcePat.test.ts": 38, + "apps/webapp/test/userActorTokenClaimsAndScopes.test.ts": 3785, + "apps/webapp/test/utils/timezones.test.ts": 14, + "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 6795, + "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 8151, + "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 7306, + "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 4339, + "apps/webapp/test/validateGitBranchName.test.ts": 4, + "apps/webapp/test/vercelUrls.test.ts": 5, + "apps/webapp/test/verifyDeploymentImage.test.ts": 612, + "apps/webapp/test/viewAsUser.test.ts": 9, + "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 7198, + "apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts": 3117, + "apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts": 5628, + "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 9896, + "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 5254, + "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 6177, + "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 4590, + "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 5203, + "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 30114, + "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 5073, + "apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts": 7181, + "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 6260, "apps/webapp/test/webhookErrorAlerts.test.ts": 4, "apps/webapp/test/workerGroupAccess.test.ts": 2, - "apps/webapp/test/workerIdUnwrap.test.ts": 47, + "apps/webapp/test/workerIdUnwrap.test.ts": 20, "apps/webapp/test/workerQueueSplit.server.test.ts": 3, - "apps/webapp/test/workerQueueSplit.test.ts": 4, - "apps/webapp/test/workerRegions.test.ts": 3, - "apps/webapp/test/workloadTokenAuthorization.test.ts": 1, - "apps/webapp/test/workloadTokenGate.integration.test.ts": 2397, - "apps/webapp/test/writableEnvironments.test.ts": 1, + "apps/webapp/test/workerQueueSplit.test.ts": 10, + "apps/webapp/test/workerRegions.test.ts": 5, + "apps/webapp/test/workloadTokenAuthorization.test.ts": 2, + "apps/webapp/test/workloadTokenGate.integration.test.ts": 2681, + "apps/webapp/test/writableEnvironments.test.ts": 3, "internal-packages/cache/src/stores/lruMemory.test.ts": 65, "internal-packages/clickhouse/src/client/client.test.ts": 7547, "internal-packages/clickhouse/src/taskRuns.test.ts": 6768, From b9f314e975d6c60bb3bd94eb0797a9486db036a4 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 22:01:52 +0100 Subject: [PATCH 12/15] test(redis-worker): stabilize shutdown timer assertion --- packages/redis-worker/src/worker.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/redis-worker/src/worker.test.ts b/packages/redis-worker/src/worker.test.ts index 16d4ffe5476..664a458735a 100644 --- a/packages/redis-worker/src/worker.test.ts +++ b/packages/redis-worker/src/worker.test.ts @@ -577,7 +577,6 @@ describe("Worker", () => { await observer.ping(); const baselineConnections = await connectedClientCount(observer); - const baselineTimeouts = activeTimeoutCount(); const worker = new Worker({ name: "shutdown-lifecycle-worker", redisOptions, @@ -604,6 +603,7 @@ describe("Worker", () => { // Let the worker enter its polling loop so the loop, rather than the deadline, wins shutdown. await new Promise((resolve) => setTimeout(resolve, 50)); + const baselineTimeouts = activeTimeoutCount(); await worker.stop(); await expect.poll(() => connectedClientCount(observer)).toBe(baselineConnections); From 90968165ae7ea1afc489c24550ff4d9eaf17f0ad Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 19 Aug 2026 22:34:37 +0100 Subject: [PATCH 13/15] test(webapp): strengthen split suite assertions --- .../apiRunListPresenter.readthrough.test.ts | 20 +++++++- .../dashboardAgentWatches.delivery.test.ts | 4 +- .../dashboardAgentWatches.lifecycle.test.ts | 1 + ...riggerTask.externalDeploymentId.helpers.ts | 48 +++++++++++++------ ...sk.externalDeploymentId.resolution.test.ts | 45 +++++++++++------ 5 files changed, 86 insertions(+), 32 deletions(-) diff --git a/apps/webapp/test/apiRunListPresenter.readthrough.test.ts b/apps/webapp/test/apiRunListPresenter.readthrough.test.ts index 5cd306cf3f9..84809630fb7 100644 --- a/apps/webapp/test/apiRunListPresenter.readthrough.test.ts +++ b/apps/webapp/test/apiRunListPresenter.readthrough.test.ts @@ -47,7 +47,7 @@ vi.mock("~/db.server", async () => { import { createPostgresContainer, replicationContainerTest } from "@internal/testcontainers"; import { PrismaClient } from "@trigger.dev/database"; -import { setTimeout } from "node:timers/promises"; +import { z } from "zod"; import { CURRENT_API_VERSION } from "~/api/versions"; import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server"; import { createRun, mirrorParents, seedParents } from "./helpers/apiRunListPresenterTestHelpers"; @@ -102,7 +102,23 @@ describe("ApiRunListPresenter public /runs routed read-through", () => { data: { id: migratedB.id }, }); - await setTimeout(1500); + const replicatedRunsQuery = clickhouse.reader.query({ + name: "waitForApiRunListPresenterTaskRuns", + query: + "SELECT countDistinct(run_id) AS count FROM trigger_dev.task_runs_v2 WHERE run_id IN {run_ids:Array(String)}", + schema: z.object({ count: z.number() }), + params: z.object({ run_ids: z.array(z.string()) }), + }); + await vi.waitFor( + async () => { + const [error, rows] = await replicatedRunsQuery({ + run_ids: [legacyOnlyA.id, legacyOnlyB.id, migratedA.id, migratedB.id], + }); + if (error) throw error; + expect(rows?.[0]?.count).toBe(4); + }, + { timeout: 15_000, interval: 100 } + ); const presenter = new ApiRunListPresenter(prisma, prisma, { newClient: prismaNew, diff --git a/apps/webapp/test/dashboardAgentWatches.delivery.test.ts b/apps/webapp/test/dashboardAgentWatches.delivery.test.ts index d569d39039f..3a129633b4c 100644 --- a/apps/webapp/test/dashboardAgentWatches.delivery.test.ts +++ b/apps/webapp/test/dashboardAgentWatches.delivery.test.ts @@ -895,11 +895,13 @@ describe("the watch card submit", () => { // The crash state: the request record is written and the watch is live, but the // process died before the confirmation was appended. - await appendChatMessageOnce(ctx.agentDb, { + const requestAppended = await appendChatMessageOnce(ctx.agentDb, { chatId: "chat_1", userId: seeded.user.id, + organizationId: seeded.organization.id, message: { id: "watch-request:wreq_1", role: "user", parts: [] } as never, }); + expect(requestAppended).toBe(true); const created = await create({ seeded, chatId: "chat_1" }); expect(created.ok).toBe(true); if (!created.ok || !created.watching) return; diff --git a/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts b/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts index 15219ed883a..8029cae9fe3 100644 --- a/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts +++ b/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts @@ -862,6 +862,7 @@ describe("the check endpoint", () => { async function activeWatch(seeded: Seeded, spec?: WatchSpec) { const result = await create({ seeded, spec }); if (!result.ok) throw new Error(`watch not created: ${result.code}`); + if (!result.watching) throw new Error("expected an active watch"); return result; } diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts index ea1bb667c1d..b52e1fbb6b7 100644 --- a/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts @@ -17,35 +17,53 @@ import { export class RecordingExternalDeploymentCache implements ExternalDeploymentCache { readonly gets: Array<{ environmentId: string; externalId: string }> = []; - readonly writes: Array<{ externalId: string; entry: ExternalDeploymentCacheEntry }> = []; + readonly writes: Array<{ + environmentId: string; + externalId: string; + entry: ExternalDeploymentCacheEntry; + }> = []; + readonly missing: Array<{ environmentId: string; externalId: string }> = []; + private readonly entries = new Map(); - constructor(private readonly entries = new Map()) {} - - readonly missing: string[] = []; + constructor( + entries: Array<{ + environmentId: string; + externalId: string; + entry: ExternalDeploymentCacheEntry; + }> = [] + ) { + for (const { environmentId, externalId, entry } of entries) { + this.entries.set(this.key(environmentId, externalId), entry); + } + } async get(environmentId: string, externalId: string) { this.gets.push({ environmentId, externalId }); - const entry = this.entries.get(externalId); + const entry = this.entries.get(this.key(environmentId, externalId)); if (entry) { return { outcome: "deployed" as const, entry }; } - return this.missing.includes(externalId) ? { outcome: "missing" as const } : null; + return this.missing.some( + (missing) => missing.environmentId === environmentId && missing.externalId === externalId + ) + ? { outcome: "missing" as const } + : null; } - async setIfNewer( - _environmentId: string, - externalId: string, - entry: ExternalDeploymentCacheEntry - ) { - this.writes.push({ externalId, entry }); - this.entries.set(externalId, entry); + async setIfNewer(environmentId: string, externalId: string, entry: ExternalDeploymentCacheEntry) { + this.writes.push({ environmentId, externalId, entry }); + this.entries.set(this.key(environmentId, externalId), entry); + } + + async setMissing(environmentId: string, externalId: string) { + this.missing.push({ environmentId, externalId }); } - async setMissing(_environmentId: string, externalId: string) { - this.missing.push(externalId); + private key(environmentId: string, externalId: string) { + return JSON.stringify([environmentId, externalId]); } } diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts index 9b9216b15f9..b74b8c8f8b3 100644 --- a/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts @@ -40,19 +40,18 @@ describe("triggerTask external deployment id", () => { const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); - const cache = new RecordingExternalDeploymentCache( - new Map([ - [ - "commit-cached", - { - workerId: worker.worker.id, - version: worker.worker.version, - sdkVersion: "", - cliVersion: "", - }, - ], - ]) - ); + const cache = new RecordingExternalDeploymentCache([ + { + environmentId: environment.id, + externalId: "commit-cached", + entry: { + workerId: worker.worker.id, + version: worker.worker.version, + sdkVersion: "", + cliVersion: "", + }, + }, + ]); const service = createService(prisma, engine, cache); @@ -163,7 +162,19 @@ describe("triggerTask external deployment id", () => { }, }); - const service = createService(prisma, engine, new NoopExternalDeploymentCache()); + const cache = new RecordingExternalDeploymentCache([ + { + environmentId: otherEnvironment.id, + externalId: "commit-elsewhere", + entry: { + workerId: worker.worker.id, + version: worker.worker.version, + sdkVersion: "", + cliVersion: "", + }, + }, + ]); + const service = createService(prisma, engine, cache); const result = await service.call({ taskId: taskIdentifier, @@ -177,6 +188,12 @@ describe("triggerTask external deployment id", () => { expect(run.status).toBe("PENDING_VERSION"); expect(run.lockedToVersionId).toBeNull(); + expect(cache.gets).toEqual([ + { environmentId: environment.id, externalId: "commit-elsewhere" }, + ]); + expect(cache.missing).toEqual([ + { environmentId: environment.id, externalId: "commit-elsewhere" }, + ]); } ); }); From b63a8efe9be1dc575a238a108354245021a91f70 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Thu, 20 Aug 2026 06:49:09 +0100 Subject: [PATCH 14/15] fix(run-engine): avoid idle batch consumer retries --- .../run-engine/src/engine/index.ts | 18 ++++++++----- .../src/engine/tests/shutdown.test.ts | 8 ++---- .../src/fair-queue/tests/fairQueue.test.ts | 27 +++++++++++++++++++ .../src/fair-queue/workerQueue.ts | 8 +++--- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 6037e28588d..9c5db057c04 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -116,6 +116,7 @@ export class RunEngine { private heartbeatTimeouts: HeartbeatTimeouts; private repairSnapshotTimeoutMs: number; private batchQueue: BatchQueue; + private batchQueueConsumersEnabled: boolean; private workerQueueObserverAbortController?: AbortController; private quitPromise?: Promise; @@ -462,13 +463,15 @@ export class RunEngine { waitpointSystem: this.waitpointSystem, }); - // Initialize BatchQueue for DRR-based batch processing (if configured) - const startBatchQueueConsumers = options.batchQueue?.consumerEnabled ?? true; + // Initialize BatchQueue for DRR-based batch processing. Consumers start lazily when the + // process-item callback is registered; before that they cannot perform useful work. + this.batchQueueConsumersEnabled = options.batchQueue?.consumerEnabled ?? true; + const batchQueueRedis = options.batchQueue?.redis ?? options.queue.redis; this.batchQueue = new BatchQueue({ redis: { - keyPrefix: `${options.batchQueue?.redis.keyPrefix ?? ""}batch-queue:`, - ...options.batchQueue?.redis, + keyPrefix: `${batchQueueRedis.keyPrefix ?? ""}batch-queue:`, + ...batchQueueRedis, }, drr: { quantum: options.batchQueue?.drr?.quantum ?? 5, @@ -482,7 +485,7 @@ export class RunEngine { defaultConcurrency: options.batchQueue?.defaultConcurrency ?? 10, globalRateLimiter: options.batchQueue?.globalRateLimiter, workerQueueMaxDepth: options.batchQueue?.workerQueueMaxDepth, - startConsumers: startBatchQueueConsumers, + startConsumers: false, retry: options.batchQueue?.retry, tracer: options.tracer, meter: options.meter, @@ -492,7 +495,7 @@ export class RunEngine { consumerCount: options.batchQueue?.consumerCount ?? 2, drrQuantum: options.batchQueue?.drr?.quantum ?? 5, defaultConcurrency: options.batchQueue?.defaultConcurrency ?? 10, - consumersEnabled: startBatchQueueConsumers, + consumersEnabled: this.batchQueueConsumersEnabled, }); this.runAttemptSystem = new RunAttemptSystem({ @@ -1923,6 +1926,9 @@ export class RunEngine { */ setBatchProcessItemCallback(callback: ProcessBatchItemCallback): void { this.batchQueue.onProcessItem(callback); + if (this.batchQueueConsumersEnabled) { + this.batchQueue.start(); + } } /** diff --git a/internal-packages/run-engine/src/engine/tests/shutdown.test.ts b/internal-packages/run-engine/src/engine/tests/shutdown.test.ts index d12e73aee48..a0d421e3c1c 100644 --- a/internal-packages/run-engine/src/engine/tests/shutdown.test.ts +++ b/internal-packages/run-engine/src/engine/tests/shutdown.test.ts @@ -17,8 +17,8 @@ async function connectedClientCount(redis: Redis): Promise { } function engineOptions(redisOptions: RedisOptions) { - // Keep caches and disabled consumers lazy so every connection opened by this test belongs to a - // shutdown resource. The run-lock client remains eager to exercise Redlock's ownership of it. + // Keep caches and consumers lazy so every connection opened by this test belongs to a shutdown + // resource. The run-lock client remains eager to exercise Redlock's ownership of it. const lazyRedisOptions = { ...redisOptions, lazyConnect: true }; return { @@ -40,10 +40,6 @@ function engineOptions(redisOptions: RedisOptions) { runLock: { redis: redisOptions }, cache: { redis: lazyRedisOptions }, debounce: { redis: lazyRedisOptions }, - batchQueue: { - redis: lazyRedisOptions, - consumerEnabled: false, - }, machines: { defaultMachine: "small-1x" as const, machines: { diff --git a/packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts b/packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts index edf5b447c7a..18517d12434 100644 --- a/packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts +++ b/packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts @@ -211,6 +211,33 @@ class TestFairQueueHelper { describe("FairQueue", () => { let keys: FairQueueKeyProducer; + describe("worker queue lifecycle", () => { + redisTest( + "aborts a blocking pop without reconnecting the disconnected client", + { timeout: 5000 }, + async ({ redisOptions }) => { + const workerQueue = new WorkerQueueManager({ + redis: redisOptions, + keys: new DefaultFairQueueKeyProducer({ prefix: "abort-test" }), + }); + const abortController = new AbortController(); + + try { + const pop = workerQueue.blockingPop(TEST_WORKER_QUEUE_ID, 60, abortController.signal); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const startedAt = performance.now(); + abortController.abort(); + + await expect(pop).resolves.toBeNull(); + expect(performance.now() - startedAt).toBeLessThan(1000); + } finally { + await workerQueue.close(); + } + } + ); + }); + describe("basic enqueue and process", () => { redisTest( "should enqueue and process a single message", diff --git a/packages/redis-worker/src/fair-queue/workerQueue.ts b/packages/redis-worker/src/fair-queue/workerQueue.ts index b3b75e9db35..087c6aedf30 100644 --- a/packages/redis-worker/src/fair-queue/workerQueue.ts +++ b/packages/redis-worker/src/fair-queue/workerQueue.ts @@ -153,9 +153,11 @@ export class WorkerQueueManager { if (cleanup && signal) { signal.removeEventListener("abort", cleanup); } - await blockingClient.quit().catch(() => { - // Ignore quit errors (may already be disconnected) - }); + if (blockingClient.status !== "end") { + await blockingClient.quit().catch(() => { + // Ignore quit errors (may already be disconnected) + }); + } } } From 4c3d66799f8b4a5e1aa6c0d3f68e47380e469829 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Thu, 20 Aug 2026 06:58:49 +0100 Subject: [PATCH 15/15] fix(run-engine): preserve batch queue key namespace --- internal-packages/run-engine/src/engine/index.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 9c5db057c04..ccfb60ca4d6 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -469,10 +469,8 @@ export class RunEngine { const batchQueueRedis = options.batchQueue?.redis ?? options.queue.redis; this.batchQueue = new BatchQueue({ - redis: { - keyPrefix: `${batchQueueRedis.keyPrefix ?? ""}batch-queue:`, - ...batchQueueRedis, - }, + // Preserve the configured namespace so existing batch state remains addressable. + redis: batchQueueRedis, drr: { quantum: options.batchQueue?.drr?.quantum ?? 5, maxDeficit: options.batchQueue?.drr?.maxDeficit ?? 50,