From 89b7522499989d428c41aa98ed5ffef6ba01c3f8 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Thu, 6 Aug 2026 18:36:53 +0000 Subject: [PATCH 01/28] =?UTF-8?q?feat(webapp):=20tell=20the=20user=20later?= =?UTF-8?q?=20=E2=80=94=20the=20dashboard=20agent's=20Watch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Tell me when this run finishes", "ping me if that error comes back". A watch checks on its own cadence, reports once, and stops within 24 hours. The user confirms a pre-filled card, so nothing starts behind their back; the answer lands in the chat, and by email if they asked for it. Restores the feature this branch's base PR set aside, unchanged: the card and chips, the wake banner and toast, the unread badge, the checks for runs, queues, errors and health, the batch scheduler and its backstops, the alert channel and its email, and the agent's schedule_watch tool with the prompt that governs it. --- .server-changes/dashboard-agent.md | 2 + .../dashboard-agent/DashboardAgent.tsx | 168 +- .../dashboard-agent/DashboardAgentChat.tsx | 62 +- .../dashboard-agent/DashboardAgentDraft.tsx | 5 +- .../dashboard-agent/DashboardAgentHistory.tsx | 18 +- .../DashboardAgentMessages.tsx | 23 + .../dashboard-agent/DashboardAgentPanel.tsx | 187 +- .../DashboardAgentSuggestedPrompts.tsx | 2 + .../components/dashboard-agent/ReportView.tsx | 36 + .../components/dashboard-agent/WakeBanner.tsx | 156 + .../dashboard-agent/WatchButton.tsx | 45 + .../components/dashboard-agent/WatchCard.tsx | 330 ++ .../components/dashboard-agent/WatchChips.tsx | 113 + .../dashboard-agent/WatchResultBlock.tsx | 48 + .../dashboard-agent/WatchWakeToast.tsx | 133 + .../dashboard-agent/chat-layout.test.ts | 1 + .../dashboard-agent/chat-layout.tsx | 16 + .../dashboardAgentLauncher.tsx | 19 +- .../demo/fixtures/page-context.ts | 16 +- .../components/dashboard-agent/list-row.tsx | 11 +- .../dashboard-agent/message-quota.ts | 12 +- .../dashboard-agent/pending-intents.test.ts | 74 +- .../dashboard-agent/pending-intents.ts | 8 + .../dashboard-agent/report-sparkline.tsx | 7 + .../suggested-prompts/page-prompts.ts | 10 + .../suggested-prompts/prompt-chips.ts | 3 +- .../suggested-prompts/resolver.test.ts | 34 +- .../suggested-prompts/resolver.ts | 2 +- .../suggested-prompts/signal-prompts.ts | 35 +- .../components/dashboard-agent/tool-labels.ts | 1 + .../dashboard-agent/turn-error.test.ts | 2 +- .../components/dashboard-agent/turn-error.ts | 5 +- .../dashboard-agent/view-actions.test.ts | 18 + .../dashboard-agent/view-catalog.tsx | 4 + .../dashboard-agent/wake-banner.test.ts | 337 ++ .../dashboard-agent/wake-poll.test.ts | 100 + .../components/dashboard-agent/wake-poll.ts | 68 + .../dashboard-agent/watch-activity.test.ts | 136 + .../dashboard-agent/watch-activity.ts | 88 + .../dashboard-agent/watch-card.test.ts | 375 ++ .../components/dashboard-agent/watch-card.ts | 181 + .../dashboard-agent/watch-chips.test.ts | 107 + .../components/dashboard-agent/watch-chips.ts | 74 + .../dashboard-agent/watch-recommendations.ts | 78 + .../app/components/queues/queue-thresholds.ts | 2 +- .../v3/ApiAlertChannelPresenter.server.ts | 5 + .../v3/dashboardAgent/block-text.ts | 98 + .../app/presenters/v3/dashboardAgent/index.ts | 5 + .../v3/dashboardAgent/watch-wording.ts | 38 + .../v3/reports/ReportPresenter.server.ts | 2 +- .../route.tsx | 20 +- .../route.tsx | 2 + .../route.tsx | 7 +- .../route.tsx | 6 +- .../route.tsx | 29 +- ...pi.v1.dashboard-agent.alerts.$channelId.ts | 100 + .../routes/api.v1.dashboard-agent.alerts.ts | 220 ++ ....dashboard-agent.watches.$watchId.check.ts | 191 + ....dashboard-agent.watches.$watchId.fired.ts | 113 + ...oard-agent.watches.$watchId.investigate.ts | 105 + ....v1.dashboard-agent.watches.batch-check.ts | 75 + .../routes/api.v1.dashboard-agent.watches.ts | 163 + ...rd-agent.alerts.$channelId.unsubscribe.tsx | 164 + ...jectParam.env.$envParam.dashboard-agent.ts | 218 +- .../route.tsx | 13 + .../dashboardAgentAlertContext.server.ts | 57 + ...hboardAgentAlertUnsubscribeToken.server.ts | 63 + .../dashboardAgentWatchAlerts.server.ts | 291 ++ .../dashboardAgentWatchBatch.server.ts | 304 ++ .../services/dashboardAgentWatchCheckBase.ts | 106 + .../dashboardAgentWatchChecks.server.ts | 331 ++ .../app/services/dashboardAgentWatchChecks.ts | 165 + .../dashboardAgentWatchErrorChecks.ts | 72 + .../dashboardAgentWatchHealthChecks.ts | 49 + .../dashboardAgentWatchInvestigate.server.ts | 89 + .../dashboardAgentWatchQueueChecks.ts | 312 ++ .../services/dashboardAgentWatchRunChecks.ts | 240 ++ .../dashboardAgentWatchSweep.server.ts | 516 +++ .../dashboardAgentWatchToken.server.ts | 181 + .../services/dashboardAgentWatches.server.ts | 1165 ++++++ apps/webapp/app/v3/alertsWorker.server.ts | 52 + apps/webapp/app/v3/commonWorker.server.ts | 38 + .../v3/services/alerts/deliverAlert.server.ts | 9 +- .../deliverDashboardAgentWatchAlert.server.ts | 506 +++ apps/webapp/package.json | 1 + apps/webapp/seed-watch-scenarios.mts | 868 +++++ .../dashboardAgentWatchWording.test.ts.snap | 291 ++ .../webapp/test/dashboardAgentBodyCap.test.ts | 6 +- .../dashboardAgentTranscriptStore.test.ts | 25 +- .../test/dashboardAgentWakeActivity.test.ts | 88 + .../dashboardAgentWatchAlertFanout.test.ts | 228 ++ ...dashboardAgentWatchAlertOwnerScope.test.ts | 275 ++ .../dashboardAgentWatchBatchFairness.test.ts | 127 + .../dashboardAgentWatchBatchRecording.test.ts | 305 ++ .../dashboardAgentWatchCardAtomicity.test.ts | 260 ++ .../dashboardAgentWatchCardRequestId.test.ts | 163 + .../test/dashboardAgentWatchChecks.test.ts | 1002 +++++ .../dashboardAgentWatchCreationReads.test.ts | 71 + .../dashboardAgentWatchInvestigate.test.ts | 335 ++ .../dashboardAgentWatchSweepBoundary.test.ts | 316 ++ .../test/dashboardAgentWatchTenancy.test.ts | 516 +++ .../test/dashboardAgentWatchToken.test.ts | 106 + .../test/dashboardAgentWatchWording.test.ts | 176 + .../webapp/test/dashboardAgentWatches.test.ts | 3264 +++++++++++++++++ apps/webapp/test/reportHealth.test.ts | 2 +- .../src/blocks.test.ts | 19 +- .../dashboard-agent-contracts/src/blocks.ts | 50 +- .../src/contracts.test.ts | 14 +- .../dashboard-agent-contracts/src/index.ts | 1 + .../dashboard-agent-contracts/src/intent.ts | 2 + .../src/watch-wording.ts | 637 ++++ .../dashboard-agent/GUIDEBOOK.md | 485 +++ internal-packages/dashboard-agent/README.md | 9 +- .../__snapshots__/prompt-prefix.test.ts.snap | 32 +- .../dashboard-agent/src/agent-runtime.ts | 14 +- .../dashboard-agent/src/compaction.test.ts | 82 + .../dashboard-agent/src/compaction.ts | 13 +- .../src/dashboard-agent.eval.ts | 4 + .../src/dashboard-agent.test.ts | 257 +- .../dashboard-agent/src/dashboard-agent.ts | 29 +- .../dashboard-agent/src/eval-policy.ts | 4 + .../dashboard-agent/src/index.ts | 9 + .../dashboard-agent/src/step-cache.ts | 5 +- .../dashboard-agent/src/tool-alerts.ts | 108 + .../src/tool-investigations.ts | 25 + .../dashboard-agent/src/tool-schemas.ts | 74 +- .../dashboard-agent/src/tools.ts | 4 + .../dashboard-agent/src/watch-actions.test.ts | 1039 ++++++ .../dashboard-agent/src/watch-actions.ts | 884 +++++ .../dashboard-agent/src/watch-batch.ts | 187 + .../dashboard-agent/src/watch-delivery.ts | 237 ++ .../dashboard-agent/src/watch-lifecycle.ts | 184 + .../src/watch-narration.test.ts | 79 + .../dashboard-agent/src/watch-narration.ts | 99 + .../src/watch-task-adapters.ts | 167 + .../dashboard-agent/src/watch-tick.test.ts | 1408 +++++++ .../dashboard-agent/src/watch-tick.ts | 221 ++ .../dashboard-agent/src/watch-tools.ts | 23 + .../migration.sql | 2 + .../database/prisma/schema.prisma | 1 + .../emails/alert-dashboard-agent-watch.tsx | 171 + internal-packages/emails/src/index.tsx | 10 + 142 files changed, 24147 insertions(+), 139 deletions(-) create mode 100644 apps/webapp/app/components/dashboard-agent/WakeBanner.tsx create mode 100644 apps/webapp/app/components/dashboard-agent/WatchButton.tsx create mode 100644 apps/webapp/app/components/dashboard-agent/WatchCard.tsx create mode 100644 apps/webapp/app/components/dashboard-agent/WatchChips.tsx create mode 100644 apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx create mode 100644 apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx create mode 100644 apps/webapp/app/components/dashboard-agent/wake-banner.test.ts create mode 100644 apps/webapp/app/components/dashboard-agent/wake-poll.test.ts create mode 100644 apps/webapp/app/components/dashboard-agent/wake-poll.ts create mode 100644 apps/webapp/app/components/dashboard-agent/watch-activity.test.ts create mode 100644 apps/webapp/app/components/dashboard-agent/watch-activity.ts create mode 100644 apps/webapp/app/components/dashboard-agent/watch-card.test.ts create mode 100644 apps/webapp/app/components/dashboard-agent/watch-card.ts create mode 100644 apps/webapp/app/components/dashboard-agent/watch-chips.test.ts create mode 100644 apps/webapp/app/components/dashboard-agent/watch-chips.ts create mode 100644 apps/webapp/app/components/dashboard-agent/watch-recommendations.ts create mode 100644 apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts create mode 100644 apps/webapp/app/presenters/v3/dashboardAgent/index.ts create mode 100644 apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts create mode 100644 apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts create mode 100644 apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts create mode 100644 apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts create mode 100644 apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts create mode 100644 apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts create mode 100644 apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts create mode 100644 apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts create mode 100644 apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx create mode 100644 apps/webapp/app/services/dashboardAgentAlertContext.server.ts create mode 100644 apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchBatch.server.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchCheckBase.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchChecks.server.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchChecks.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchRunChecks.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchSweep.server.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatchToken.server.ts create mode 100644 apps/webapp/app/services/dashboardAgentWatches.server.ts create mode 100644 apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts create mode 100644 apps/webapp/seed-watch-scenarios.mts create mode 100644 apps/webapp/test/__snapshots__/dashboardAgentWatchWording.test.ts.snap create mode 100644 apps/webapp/test/dashboardAgentWakeActivity.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchAlertFanout.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchAlertOwnerScope.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchBatchFairness.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchBatchRecording.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchCardAtomicity.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchCardRequestId.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchChecks.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchCreationReads.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchInvestigate.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchSweepBoundary.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchTenancy.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchToken.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatchWording.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatches.test.ts create mode 100644 internal-packages/dashboard-agent-contracts/src/watch-wording.ts create mode 100644 internal-packages/dashboard-agent/GUIDEBOOK.md create mode 100644 internal-packages/dashboard-agent/src/tool-alerts.ts create mode 100644 internal-packages/dashboard-agent/src/watch-actions.test.ts create mode 100644 internal-packages/dashboard-agent/src/watch-actions.ts create mode 100644 internal-packages/dashboard-agent/src/watch-batch.ts create mode 100644 internal-packages/dashboard-agent/src/watch-delivery.ts create mode 100644 internal-packages/dashboard-agent/src/watch-lifecycle.ts create mode 100644 internal-packages/dashboard-agent/src/watch-narration.test.ts create mode 100644 internal-packages/dashboard-agent/src/watch-narration.ts create mode 100644 internal-packages/dashboard-agent/src/watch-task-adapters.ts create mode 100644 internal-packages/dashboard-agent/src/watch-tick.test.ts create mode 100644 internal-packages/dashboard-agent/src/watch-tick.ts create mode 100644 internal-packages/dashboard-agent/src/watch-tools.ts create mode 100644 internal-packages/database/prisma/migrations/20260729120000_add_dashboard_agent_watch_alert_type/migration.sql create mode 100644 internal-packages/emails/emails/alert-dashboard-agent-watch.tsx diff --git a/.server-changes/dashboard-agent.md b/.server-changes/dashboard-agent.md index 504bb70c4cf..e40ae2076b9 100644 --- a/.server-changes/dashboard-agent.md +++ b/.server-changes/dashboard-agent.md @@ -7,6 +7,8 @@ Meet the dashboard agent: a chat in every environment that answers questions abo **Investigate** on a failed run, an error, a backed-up queue or a run that hasn't started gets you a worked-through answer — what happened, why, and how to fix it, with every claim linked to the runs, errors and deploys behind it. +**Watch…** on a run, queue, error or the health report tells you when things change: a run finishes, a queue clears or grows past a number you pick, an error comes back, an environment recovers. The answer arrives in the chat and, if you want, by email, Slack or webhook — and the agent can look into bad news on its own. A watch reaches you on any browser you sign in from, without opening the chat first. + The health report reads the same everywhere — dashboard, terminal, editor. A very long chat keeps working: the agent summarises the earlier part and carries on. The agent's replies no longer show images. A sample of conversations is scored automatically so the agent keeps getting better. Only the score and a one-line summary are kept, never your messages, data or code, and we can switch it off for your organization on request. diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx index e7967b2ac38..9b58f5500a5 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx @@ -1,4 +1,4 @@ -import type { SuggestedPrompt } from "@internal/dashboard-agent-contracts"; +import type { SuggestedPrompt, WatchSpec } from "@internal/dashboard-agent-contracts"; import { useLocation } from "@remix-run/react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { @@ -6,6 +6,9 @@ import { ResizablePanel, ResizablePanelGroup, } from "~/components/primitives/Resizable"; +import { useEnvironment } from "~/hooks/useEnvironment"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; import { useShortcutKeys } from "~/hooks/useShortcutKeys"; import { DashboardAgentPanel } from "./DashboardAgentPanel"; import { DashboardAgentProvider, TOGGLE_PANEL_SHORTCUT } from "./dashboardAgentLauncher"; @@ -16,18 +19,68 @@ import { readAgentFullscreen, writeAgentFullscreen, } from "./panel-layout"; +import { startWakePolling } from "./wake-poll"; +import { shouldPollWakeFeed, subscribeWatchActivity } from "./watch-activity"; +import { + showWatchWakesSummaryToast, + showWatchWakeToast, + WAKE_TOAST_MAX_INDIVIDUAL, + type WatchWake, +} from "./WatchWakeToast"; + +const TOASTED_WAKES_STORAGE_KEY = "tdev:dashboard-agent:toasted-wakes"; + +// Shorter than the poll interval, so a stuck request is dropped before the next tick. +const UNREAD_REQUEST_TIMEOUT_MS = 30_000; /** `hasAccess` is a UI gate only; the resource routes enforce the same check server-side. */ export function DashboardAgent({ children, hasAccess = false, promotedPrompt, + /** From the page load: unread wakes waiting for this user, whatever this browser remembers. */ + initialUnreadWakes = 0, + /** Also from the page load: a watch is running, so a wake can still arrive in this tab. */ + hasActiveWatches = false, }: { children: React.ReactNode; hasAccess?: boolean; promotedPrompt?: SuggestedPrompt; + initialUnreadWakes?: number; + hasActiveWatches?: boolean; }) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + const actionPath = `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/dashboard-agent`; + const [open, setOpen] = useState(false); + // Seeded from the page load, so the launcher dot is right before the first poll answers. + const [unreadWakes, setUnreadWakes] = useState(initialUnreadWakes); + const toastedWakes = useRef(new Set()); + // The toast source is recent deliveries, not unread, so the dedupe must survive a reload. + useEffect(() => { + try { + const raw = window.localStorage.getItem(TOASTED_WAKES_STORAGE_KEY); + if (raw) for (const id of JSON.parse(raw) as string[]) toastedWakes.current.add(id); + } catch { + // Storage unavailable; the in-memory dedupe still applies. + } + }, []); + const rememberToasted = useCallback((watchId: string) => { + toastedWakes.current.add(watchId); + try { + // Newest ids only, so the key can't grow unbounded. + window.localStorage.setItem( + TOASTED_WAKES_STORAGE_KEY, + JSON.stringify([...toastedWakes.current].slice(-50)) + ); + } catch { + // Same as the read. + } + }, []); + // A wake in the on-screen chat toasts but must not light the dot. + const visibleChat = useRef(null); // Read lazily so SSR always renders the side panel. const [fullscreen, setFullscreen] = useState(readAgentFullscreen); @@ -53,17 +106,32 @@ export function DashboardAgent({ const [requestedMessage, setRequestedMessage] = useState< { text: string; seq: number } | undefined >(undefined); + // `seq` so the same chat can be asked for twice. + const [openChatRequest, setOpenChatRequest] = useState< + { chatId: string; seq: number } | undefined + >(undefined); + const [watchRequest, setWatchRequest] = useState<{ spec: WatchSpec; seq: number } | undefined>( + undefined + ); const setPanelOpen = useCallback((next: boolean) => { setOpen(next); // Pending requests must be dropped or a stale one re-applies on the next open. if (!next) { + visibleChat.current = null; setFullscreen(false); writeAgentFullscreen(false); setRequestedMessage(undefined); + setOpenChatRequest(undefined); + setWatchRequest(undefined); } }, []); + const openChat = useCallback((chatId: string) => { + setOpen(true); + setOpenChatRequest((current) => ({ chatId, seq: (current?.seq ?? 0) + 1 })); + }, []); + const openWith = useCallback((text: string) => { const trimmed = text.trim(); if (!trimmed) return; @@ -71,6 +139,97 @@ export function DashboardAgent({ setRequestedMessage((current) => ({ text: trimmed, seq: (current?.seq ?? 0) + 1 })); }, []); + const openWithWatch = useCallback((spec: WatchSpec) => { + setOpen(true); + setWatchRequest((current) => ({ spec, seq: (current?.seq ?? 0) + 1 })); + }, []); + + // Nothing to be woken about means nothing to poll for. The page load's unread count and + // active-watch flag are the ungated signals; the browser's own memory of a watch starts the + // poll without a reload. Once any says yes this tab keeps polling, so a wake reaches a tab + // that was open before the watch existed. + const [watching, setWatching] = useState(false); + useEffect(() => { + const sync = () => { + if ( + shouldPollWakeFeed({ + serverUnreadWakes: initialUnreadWakes, + serverHasActiveWatches: hasActiveWatches, + organizationId: organization.id, + }) + ) + setWatching(true); + }; + sync(); + return subscribeWatchActivity(sync); + }, [organization.id, initialUnreadWakes, hasActiveWatches]); + + useEffect(() => { + if (!hasAccess || !watching) return; + + let cancelled = false; + const load = async () => { + try { + // Bounded, so one stuck request can't hold the poll's in-flight guard. + const res = await fetch(`${actionPath}?unread=1`, { + signal: AbortSignal.timeout(UNREAD_REQUEST_TIMEOUT_MS), + }); + if (!res.ok) return; + const data = (await res.json()) as { unreadWakes?: number; wakes?: WatchWake[] }; + if (cancelled) return; + // The wakes list carries read ones too, so only unread ones are subtracted. + const unreadInView = (data.wakes ?? []).filter( + (wake) => wake.unread && wake.chatId === visibleChat.current + ).length; + setUnreadWakes(Math.max(0, (data.unreadWakes ?? 0) - unreadInView)); + + const fresh = (data.wakes ?? []).filter((wake) => !toastedWakes.current.has(wake.watchId)); + for (const wake of fresh) rememberToasted(wake.watchId); + + if (fresh.length > WAKE_TOAST_MAX_INDIVIDUAL) { + showWatchWakesSummaryToast(fresh.length, () => setPanelOpen(true)); + } else { + for (const wake of [...fresh].reverse()) { + showWatchWakeToast(wake, openChat); + } + } + } catch { + // Try again next tick. + } + }; + + const stop = startWakePolling({ + load, + isHidden: () => document.hidden, + onVisibilityChange: (listener) => { + document.addEventListener("visibilitychange", listener); + return () => document.removeEventListener("visibilitychange", listener); + }, + }); + + return () => { + cancelled = true; + stop(); + }; + }, [hasAccess, watching, actionPath, setPanelOpen, openChat]); + + // Zeroes the dot right away; the poll restores the truth if another chat has one. + const markChatRead = useCallback( + async (chatId: string) => { + visibleChat.current = chatId; + setUnreadWakes(0); + const body = new FormData(); + body.set("intent", "read"); + body.set("chatId", chatId); + try { + await fetch(actionPath, { method: "POST", body }); + } catch { + // Catches up on the next open. + } + }, + [actionPath] + ); + // ⌘J is contextual: closed opens the panel, open starts a new chat. It never closes. useShortcutKeys({ shortcut: TOGGLE_PANEL_SHORTCUT, @@ -88,8 +247,8 @@ export function DashboardAgent({ useDashboardAgentOpenRequests({ enabled: hasAccess, openWith, setOpen: setPanelOpen }); const context = useMemo( - () => ({ open, setOpen: setPanelOpen, openWith }), - [open, setPanelOpen, openWith] + () => ({ open, setOpen: setPanelOpen, openWith, openWithWatch, unreadWakes }), + [open, setPanelOpen, openWith, openWithWatch, unreadWakes] ); if (!hasAccess) { @@ -118,8 +277,11 @@ export function DashboardAgent({ setPanelOpen(false)} requestedMessage={requestedMessage} + openChatRequest={openChatRequest} + watchRequest={watchRequest} newChatSeq={newChatSeq} promotedPrompt={promotedPrompt} + onChatRead={markChatRead} isFullscreen={fullscreen} onToggleFullscreen={toggleFullscreen} /> diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx index 526dedde342..11261222d5c 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx @@ -1,7 +1,12 @@ import { useChat } from "@ai-sdk/react"; import type { UIMessage } from "@ai-sdk/react"; import type { dashboardAgent } from "@internal/dashboard-agent"; -import type { AgentIntent, SuggestedPrompt } from "@internal/dashboard-agent-contracts"; +import { + isWatchRequestMessageId, + type AgentIntent, + type SuggestedPrompt, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; import { useNavigate } from "@remix-run/react"; import { useTriggerChatTransport } from "@trigger.dev/sdk/chat/react"; import { useCallback, useEffect, useRef, useState } from "react"; @@ -14,7 +19,7 @@ import { DashboardAgentMessages, type TurnActivity } from "./DashboardAgentMessa import { MESSAGE_TOO_LARGE_ERROR } from "./message-limits"; import { createTranscriptOrder, orderTranscript } from "./message-order"; import { appendRunFilters } from "./navigate-target"; -import { pendingNavigateIntents } from "./pending-intents"; +import { pendingNavigateIntents, pendingWatchIntents } from "./pending-intents"; import type { AgentPageContext } from "./page-context-types"; import { fetchChatTranscript, @@ -23,6 +28,7 @@ import { } from "./settled-transcript"; import { useAgentMessageQuota } from "./useAgentMessageQuota"; import { useTriggerUriResolver } from "./useTriggerUriResolver"; +import { WatchChips, type WatchChip } from "./WatchChips"; // Resuming with `lastEventId` stops the `.out` stream replaying the previous turn. export type DashboardAgentSession = { @@ -55,7 +61,12 @@ export function DashboardAgentChat({ streaming, prefill, promotedPrompt, + watches, pagePaths, + watchCard, + appendedMessages, + onWatchIntent, + onCancelWatch, onTurnSettled, onActivityChange, }: { @@ -75,7 +86,13 @@ export function DashboardAgentChat({ // `seq` makes each request distinct so the same text can be sent twice. prefill?: { text: string; seq: number }; promotedPrompt?: SuggestedPrompt; + watches: WatchChip[]; pagePaths?: Record; + watchCard?: React.ReactNode; + appendedMessages?: { messages: UIMessage[]; seq: number }; + /** Nothing is persisted until the user submits the card. */ + onWatchIntent?: (spec: WatchSpec) => void; + onCancelWatch: (watchId: string) => void; onTurnSettled: () => void; onActivityChange?: (chatId: string, activity: TurnActivity | null) => void; }) { @@ -172,6 +189,20 @@ export function DashboardAgentChat({ const activity: TurnActivity | null = status === "submitted" ? "thinking" : status === "streaming" ? "working" : null; + // Once per `seq`: the append is already persisted, so a replay would duplicate it. + // Ids are stable, so anything already in the transcript is skipped. + const appendedSeq = useRef(undefined); + useEffect(() => { + if (!appendedMessages || appendedSeq.current === appendedMessages.seq) return; + appendedSeq.current = appendedMessages.seq; + setMessages((current) => { + const missing = appendedMessages.messages.filter( + (message) => !current.some((existing) => existing.id === message.id) + ); + return missing.length === 0 ? current : [...current, ...missing]; + }); + }, [appendedMessages, setMessages]); + const sentFirst = useRef(false); useEffect(() => { if (pendingFirstMessage && !sentFirst.current) { @@ -192,7 +223,10 @@ export function DashboardAgentChat({ ); const retry = useCallback(() => { - const lastUserMessage = [...messages].reverse().find((m) => m.role === "user"); + // A watch's consent record is a user message nobody typed, so retry skips it. + const lastUserMessage = [...messages] + .reverse() + .find((m) => m.role === "user" && !isWatchRequestMessageId(m.id)); const text = lastUserMessage?.parts ?.filter((p): p is { type: "text"; text: string } => p.type === "text") .map((p) => p.text) @@ -230,6 +264,9 @@ export function DashboardAgentChat({ case "ask": submit(intent.prompt); return; + case "watch": + onWatchIntent?.(intent.spec); + return; case "navigate": void goTo(intent); return; @@ -237,7 +274,7 @@ export function DashboardAgentChat({ console.warn(`Dashboard agent: unhandled intent "${intent.kind}"`); } }, - [submit, goTo] + [submit, goTo, onWatchIntent] ); // Seeded from the loaded transcript before first render, so history never re-navigates. @@ -252,6 +289,17 @@ export function DashboardAgentChat({ if (target) void goTo(target); }, [messages, goTo]); + const watchProposedRef = useRef | null>(null); + if (watchProposedRef.current === null) { + watchProposedRef.current = new Set(); + pendingWatchIntents(initialMessages, watchProposedRef.current); + } + useEffect(() => { + const pending = pendingWatchIntents(messages, watchProposedRef.current!); + const proposed = pending.at(-1); + if (proposed) onWatchIntent?.(proposed.spec); + }, [messages, onWatchIntent]); + const stop = useCallback(() => { transport.stopGeneration(chatId); aiStop(); @@ -286,6 +334,10 @@ export function DashboardAgentChat({ return ( <> + watch.status === "active")} + onCancel={onCancelWatch} + /> {messages.length === 0 && !pendingFirstMessage ? ( )} + {watchCard ?
{watchCard}
: null} {quota.kind === "reached" ? ( void; projectSlug: string; @@ -22,6 +23,7 @@ export function DashboardAgentDraft({ currentPage: string; pageContext?: AgentPageContext; promotedPrompt?: SuggestedPrompt; + watchCard?: React.ReactNode; }) { const [input, setInput] = useState(""); @@ -56,6 +58,7 @@ export function DashboardAgentDraft({ promoted={promotedPrompt} composer={
+ {watchCard} submit(input)} onStop={() => {}} isStreaming={false} - placeholderSuggestion={placeholderSuggestion} + placeholderSuggestion={watchCard ? undefined : placeholderSuggestion} context={ = { thinking: "Agent is thinking", investigating: "Investigation in progress", + watching: "Watch active", }; function chatProcess(chat: DashboardAgentChat, isThinking: boolean): ChatProcess | null { if (isThinking) return "thinking"; if (chat.hasOpenInvestigation) return "investigating"; + if (chat.hasActiveWatch) return "watching"; return null; } @@ -43,6 +49,13 @@ function ProcessIcon({ process }: { process: ChatProcess }) { ); } +// Must stay a stable sort on one key: everything else keeps the server's order. +function unreadFirst(chats: DashboardAgentChat[]): DashboardAgentChat[] { + return [...chats].sort( + (a, b) => Number(b.hasUnreadWake ?? false) - Number(a.hasUnreadWake ?? false) + ); +} + // Weeks are the coarsest unit: months render as "1.8mo" for eight weeks. const AGE_UNITS = ["w", "d", "h", "m"] as const; @@ -84,13 +97,14 @@ export function DashboardAgentHistoryMenu({ ) : ( - {chats.map((chat) => { + {unreadFirst(chats).map((chat) => { const process = chatProcess(chat, chat.id === thinkingChatId); const age = chat.lastMessageAt ? chatAge(chat.lastMessageAt, now) : undefined; return ( : null} meta={age} variant={chat.id === currentChatId ? "selected" : "default"} diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx index 01bd9441d52..f255d29c2b2 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx @@ -16,6 +16,7 @@ import { ChatText, ChatTranscript, ChatTurn, + ChatWakeSlot, } from "./chat-layout"; import { reuseWinners } from "./investigation-winners"; import { stripModelImages } from "./model-markdown"; @@ -23,6 +24,7 @@ import { reportBlockFromToolPart } from "./report-block-adapter"; import { shouldShowLiveTurnError } from "./turn-error"; import type { ResolvedUri } from "./ReportView"; import { ViewBlocks } from "./view-catalog"; +import { findWakeWatch, WakeBanner, wakeRefFromMessageId, type WakeWatch } from "./WakeBanner"; export type { TurnActivity }; @@ -35,6 +37,8 @@ export type DashboardAgentMessagesProps = { onIntent?: (intent: AgentIntent) => void; resolveUri?: (uri: string) => ResolvedUri | null; pagePaths?: Record; + /** Optional: without it a wake banner falls back to kind-agnostic wording. */ + watches?: WakeWatch[]; }; // Returns the same reference when there are no `step-start` parts, so memoization holds. @@ -201,12 +205,14 @@ const DashboardAgentTurn = memo(function DashboardAgentTurn({ onIntent, resolveUri, pagePaths, + watches, investigationWinners, }: { message: UIMessage; onIntent?: (intent: AgentIntent) => void; resolveUri?: (uri: string) => ResolvedUri | null; pagePaths?: Record; + watches?: WakeWatch[]; /** See {@link winningInvestigationOccurrences}. */ investigationWinners?: Map; }) { @@ -264,6 +270,21 @@ const DashboardAgentTurn = memo(function DashboardAgentTurn({ body.push(renderDashboardPart(part, i, resolveUri)); } + const wake = wakeRefFromMessageId(message.id); + if (wake) { + return ( + + + } + > + {body} + + + ); + } + return {body}; }); @@ -276,6 +297,7 @@ export function DashboardAgentTurns({ onIntent, resolveUri, pagePaths, + watches, }: DashboardAgentMessagesProps) { // Must be the exact parts the turns render: the winners map keys by part index. const stripped = useMemo(() => messages.map(stripStepParts), [messages]); @@ -296,6 +318,7 @@ export function DashboardAgentTurns({ onIntent={onIntent} resolveUri={resolveUri} pagePaths={pagePaths} + watches={watches} investigationWinners={investigationWinners} /> ))} diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx index 64c14afc68c..0a613cc50de 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx @@ -16,10 +16,13 @@ import { type DashboardAgentSession, } from "./DashboardAgentChat"; import { DashboardAgentDraft } from "./DashboardAgentDraft"; +import { WatchCard } from "./WatchCard"; +import { watchDraftFor } from "./watch-card"; +import { forgetWatchActivity, rememberWatchActivity } from "./watch-activity"; import type { TurnActivity } from "./DashboardAgentMessages"; import { DashboardAgentHeader } from "./DashboardAgentHeader"; import type { DashboardAgentChat as DashboardAgentChatListItem } from "./DashboardAgentHistory"; -import type { SuggestedPrompt } from "@internal/dashboard-agent-contracts"; +import type { SuggestedPrompt, WatchDraft, WatchSpec } from "@internal/dashboard-agent-contracts"; import type { AgentPageContext } from "./page-context-types"; import { agentPageLabel } from "./page-label"; import { AgentPanelColumn } from "./panel-layout"; @@ -63,8 +66,11 @@ type ActiveChat = { export function DashboardAgentPanel({ onClose, requestedMessage, + openChatRequest, newChatSeq, promotedPrompt, + watchRequest, + onChatRead, isFullscreen = false, onToggleFullscreen, }: { @@ -73,8 +79,11 @@ export function DashboardAgentPanel({ onToggleFullscreen?: () => void; // Every `seq` below distinguishes repeat requests with identical contents. requestedMessage?: { text: string; seq: number }; + openChatRequest?: { chatId: string; seq: number }; newChatSeq?: number; promotedPrompt?: SuggestedPrompt; + watchRequest?: { spec: WatchSpec; seq: number }; + onChatRead?: (chatId: string) => void; }) { const organization = useOrganization(); const project = useProject(); @@ -126,6 +135,9 @@ export function DashboardAgentPanel({ const historyInFlight = useRef | null>(null); + // The read POST and its reload can land out of order, so mask the next list. + const justRead = useRef>(new Set()); + const loadHistory = useCallback(async () => { if (historyInFlight.current) return historyInFlight.current; const request = (async () => { @@ -133,7 +145,17 @@ export function DashboardAgentPanel({ const res = await fetch(actionPath); if (!res.ok) throw new Error(`History request failed (${res.status})`); const data = (await res.json()) as { chats?: DashboardAgentChatListItem[] }; - setChats(data.chats ?? []); + const read = justRead.current; + justRead.current = new Set(); + const chats = data.chats ?? []; + // Reloaded after every turn and after a watch is created, so this is where the browser + // learns whether the wake feed is worth polling. + const pending = chats.some((chat) => chat.hasActiveWatch || chat.hasUnreadWake); + if (pending) rememberWatchActivity(organization.id); + else forgetWatchActivity(organization.id); + setChats( + chats.map((chat) => (read.has(chat.id) ? { ...chat, hasUnreadWake: false } : chat)) + ); } catch (error) { console.error("Dashboard agent: failed to load chat history", error); toast.error("We couldn't load your previous chats. Try again in a moment."); @@ -143,7 +165,7 @@ export function DashboardAgentPanel({ })(); historyInFlight.current = request; return request; - }, [actionPath, toast]); + }, [actionPath, organization.id, toast]); // Bumped on each open so a slower earlier open can't overwrite a newer one. const openChatRequestSeq = useRef(0); @@ -259,10 +281,22 @@ export function DashboardAgentPanel({ openChatRequestSeq.current += 1; setActive(null); setLoading(false); + setWatchDraft(null); setChats([]); void loadHistory(); }, [organization.id, loadHistory]); + const handledOpenChatSeq = useRef(undefined); + useEffect(() => { + if (!openChatRequest || handledOpenChatSeq.current === openChatRequest.seq) return; + handledOpenChatSeq.current = openChatRequest.seq; + // Reloading the visible transcript would drop a turn in flight. + if (openChatRequest.chatId === active?.chatId) return; + void openChat(openChatRequest.chatId); + // `active` is read, not tracked: a later change must not re-run the request. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [openChatRequest, openChat]); + useEffect(() => { if (!active?.chatId) return; try { @@ -275,6 +309,21 @@ export function DashboardAgentPanel({ } }, [active?.chatId, storageKey, location.pathname]); + useEffect(() => { + if (!active?.chatId) return; + const chatId = active.chatId; + onChatRead?.(chatId); + justRead.current.add(chatId); + setChats((previous) => + previous.map((chat) => (chat.id === chatId ? { ...chat, hasUnreadWake: false } : chat)) + ); + // Read again on the way out: a wake can land while the chat is open. + return () => { + onChatRead?.(chatId); + justRead.current.add(chatId); + }; + }, [active?.chatId, onChatRead]); + // Bound to its chat, which remounts with a fresh guard ref on every switch. const [prefill, setPrefill] = useState<{ text: string; seq: number; chatId: string } | undefined>( undefined @@ -291,6 +340,100 @@ export function DashboardAgentPanel({ } }, [requestedMessage, loading, active, createChat]); + const [watchDraft, setWatchDraft] = useState(null); + const [watchPending, setWatchPending] = useState(false); + const [watchError, setWatchError] = useState(null); + // Carries its chat id so a later-mounted chat cannot adopt another chat's block. + const [appendedMessages, setAppendedMessages] = useState< + { chatId: string; messages: UIMessage[]; seq: number } | undefined + >(undefined); + // One id per configured card, kept across retries: the server writes the request + // and confirmation records under it, so retrying repairs instead of duplicating. + const watchRequestId = useRef(undefined); + + const handledWatchSeq = useRef(undefined); + useEffect(() => { + if (!watchRequest || handledWatchSeq.current === watchRequest.seq) return; + handledWatchSeq.current = watchRequest.seq; + setWatchError(null); + setWatchPending(false); + watchRequestId.current = generateFriendlyId("wreq"); + setWatchDraft(watchDraftFor(watchRequest.spec)); + }, [watchRequest]); + + // Nothing is posted or persisted until the card is submitted. + const openWatchCard = useCallback((spec: WatchSpec) => { + setWatchError(null); + setWatchPending(false); + watchRequestId.current = generateFriendlyId("wreq"); + setWatchDraft(watchDraftFor(spec)); + }, []); + + const dismissWatchCard = useCallback(() => { + watchRequestId.current = undefined; + setWatchDraft(null); + setWatchError(null); + setWatchPending(false); + }, []); + + const submitWatch = useCallback(async () => { + if (!watchDraft) return; + setWatchPending(true); + setWatchError(null); + try { + const body = new FormData(); + body.set("intent", "watch-create"); + body.set("draft", JSON.stringify(watchDraft)); + // Held across retries, so a resubmit repairs the same pair of records. + watchRequestId.current ??= generateFriendlyId("wreq"); + body.set("clientRequestId", watchRequestId.current); + // A watch is chat-bound: with no chat open the server creates one. + if (active?.chatId) body.set("chatId", active.chatId); + + const res = await fetch(actionPath, { method: "POST", body }); + const data = (await res.json()) as { + chatId?: string; + messages?: UIMessage[]; + error?: string; + }; + if (!res.ok || !data.chatId || !data.messages) { + setWatchError(data.error ?? "We couldn't start that watch. Try again in a moment."); + return; + } + + const messages = data.messages; + if (active?.chatId === data.chatId) { + setAppendedMessages((current) => ({ + chatId: data.chatId!, + messages, + seq: (current?.seq ?? 0) + 1, + })); + } else { + // No session: nothing is streaming and the records are the whole chat. + setActive({ chatId: data.chatId, messages, session: null }); + } + watchRequestId.current = undefined; + setWatchDraft(null); + void loadHistory(); + } catch (error) { + console.error("Dashboard agent: failed to create watch", error); + setWatchError("We couldn't start that watch. Try again in a moment."); + } finally { + setWatchPending(false); + } + }, [watchDraft, active?.chatId, actionPath, loadHistory]); + + const watchCard = watchDraft ? ( + void submitWatch()} + onCancel={dismissWatchCard} + pending={watchPending} + error={watchError} + /> + ) : null; + const newChat = useCallback(() => { // Invalidate any in-flight open or create so its result can't replace the draft. openChatRequestSeq.current += 1; @@ -333,10 +476,40 @@ export function DashboardAgentPanel({ [actionPath, active?.chatId, newChat, loadHistory, toast] ); + const cancelWatch = useCallback( + async (watchId: string) => { + const chatId = active?.chatId; + if (!chatId) return; + setChats((previous) => + previous.map((chat) => + chat.id === chatId + ? { ...chat, watches: (chat.watches ?? []).filter((watch) => watch.id !== watchId) } + : chat + ) + ); + const body = new FormData(); + body.set("intent", "watch-cancel"); + body.set("chatId", chatId); + body.set("watchId", watchId); + try { + const res = await fetch(actionPath, { method: "POST", body }); + if (!res.ok) throw new Error(`Watch cancel failed (${res.status})`); + } catch (error) { + console.error("Dashboard agent: failed to cancel watch", error); + toast.error("We couldn't stop that watch. Try again in a moment."); + } + void loadHistory(); + }, + [actionPath, active?.chatId, loadHistory, toast] + ); + // Titles are written when the first turn settles, so a new chat has none yet. const activeChat = active ? chats.find((chat) => chat.id === active.chatId) : undefined; const headerTitle = active ? (activeChat?.title ?? "Chat") : "New chat"; + // Not filtered to active: the wake banner needs watches that already fired. + const chatWatches = activeChat?.watches ?? []; + return (
)} diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx index c7f935b1d9b..b9a53192ba9 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx @@ -1,6 +1,7 @@ import { BookOpenIcon, ChartBarIcon, + EyeIcon, MagnifyingGlassIcon, QuestionMarkCircleIcon, SparklesIcon, @@ -22,6 +23,7 @@ export const PROMPT_SLOT_BUTTON: Record< > = { promoted: { variant: "primary/small", icon: SparklesIcon }, investigate: { variant: "primary/small", icon: MagnifyingGlassIcon }, + watch: { variant: "secondary/small", icon: EyeIcon }, status: { variant: "secondary/small", icon: ChartBarIcon }, explain: { variant: "tertiary/small", icon: QuestionMarkCircleIcon }, docs: { variant: "docs/small", icon: BookOpenIcon }, diff --git a/apps/webapp/app/components/dashboard-agent/ReportView.tsx b/apps/webapp/app/components/dashboard-agent/ReportView.tsx index 936d7b319da..562cbe40430 100644 --- a/apps/webapp/app/components/dashboard-agent/ReportView.tsx +++ b/apps/webapp/app/components/dashboard-agent/ReportView.tsx @@ -31,6 +31,7 @@ import { import { type ReportMessages } from "~/presenters/v3/reports/report-messages"; import { AgentBadge } from "./agent-badges"; import { + FOOTER_WATCH_CODE, ReportBody, ReportCard, ReportFindingLine, @@ -52,6 +53,9 @@ import { export type ResolvedUri = { label: string; url: string }; +/** How often a recovery watch polls, and how long it lives. Aggregate conditions floor at 5m. */ +const RECOVERY_WATCH = { checkEveryMinutes: 5, maxHours: 6 } as const; + // --- messages --------------------------------------------------------------- /** @@ -295,6 +299,22 @@ export function ReportView({ const linkByKey = (key: string | undefined) => key === undefined ? undefined : vm.links.find((link) => link.key === key)?.url; + // Only offered when there is something to recover from, and only for the health + // report, which is the one with a recovery watch kind. + const recoveryWatch: AgentIntent | null = + vm.title === "health" && (severity === "warn" || severity === "crit") + ? { + kind: "watch", + spec: { + kind: "health_recovery", + report: "health", + fromSeverity: severity, + note: `${vm.scope} health back to normal`, + ...RECOVERY_WATCH, + }, + } + : null; + // Links a footer action already speaks for aren't repeated as reading matter. const footerLinkKeys = new Set(layout.footer.map((entry) => entry.link).filter(Boolean)); @@ -309,6 +329,22 @@ export function ReportView({ }), })); + if (recoveryWatch && onIntent) { + const watchItem: ReportFooterItem = { + code: FOOTER_WATCH_CODE, + // The label is deliberately the same everywhere; only the pre-filled spec is + // contextual, so a per-object label would break the pattern. + node: onIntent(recoveryWatch)}>Watch…, + }; + // The watch joins the other buttons, before the trailing prose entry. + const noteIndex = footerItems.findIndex((item) => reportFooterStyle(item.code) === "note"); + if (noteIndex !== -1) { + footerItems.splice(noteIndex, 0, watchItem); + } else { + footerItems.push(watchItem); + } + } + // Resources the report cites, resolved to dashboard links by the host. Cited, // not offered, so a text link; our docs still get the docs button. for (const link of vm.links) { diff --git a/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx b/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx new file mode 100644 index 00000000000..0a54ddd894f --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx @@ -0,0 +1,156 @@ +/** + * The banner above a wake narration. + * + * This component holds no kind-specific wording: tone, semantic icon and headline + * come from contracts and `app/presenters/v3/dashboardAgent`. All it decides is which glyph a + * semantic icon draws and which frame a tone paints. + * + * A wake is identified by its message id, `wake:watch:{watchId}:{fired|expired}`. + * That suffix is the transport encoding, not the outcome; the outcome comes off the + * watch row. + */ +import { + CheckCircleIcon, + ClockIcon, + ExclamationCircleIcon, + ExclamationTriangleIcon, + InformationCircleIcon, +} from "@heroicons/react/20/solid"; +import type { + WatchObservedOutcome, + WatchResolution, + WatchSemanticIcon, +} from "@internal/dashboard-agent-contracts"; +import { cn } from "~/utils/cn"; +import { type AgentTone, TONE_ICON_COLOR } from "./agent-badges"; +import { + presentResolvedWatch, + watchSubline, + WATCH_PRESENTATION_FALLBACK, +} from "~/presenters/v3/dashboardAgent"; + +const WAKE_ID_PREFIX = "wake:watch:"; + +/** + * The wire encoding in a wake's message id, not the resolution: `window_completed` + * and `condition_impossible` are both addressed as `expired`, and the row is the + * authority on which one it was. + */ +export type WakeOutcome = "fired" | "expired"; + +/** The watch fields a banner can use. A `WatchChip` satisfies it. */ +export type WakeWatch = { + id: string; + kind: string; + note: string; + identity: string; + /** How the watch ended. Absent on a row written before the resolution model. */ + resolution?: WatchResolution | null; + /** What the resolving check observed — the other half of the headline. */ + observedOutcome?: WatchObservedOutcome | null; + /** + * Why the watch ended, from its last result. Only used to reconstruct a + * resolution for rows that predate the `resolution` column. + */ + endedReason?: string | null; +}; + +export type WakeRef = { watchId: string; outcome: WakeOutcome }; + +/** + * The watch a message narrates the wake of, or null when the message isn't a wake. + * A watch id never ends in an outcome word, so splitting on the last colon is + * unambiguous. + */ +export function wakeRefFromMessageId(messageId: string): WakeRef | null { + if (!messageId.startsWith(WAKE_ID_PREFIX)) return null; + const rest = messageId.slice(WAKE_ID_PREFIX.length); + const split = rest.lastIndexOf(":"); + if (split <= 0) return null; + const outcome = rest.slice(split + 1); + if (outcome !== "fired" && outcome !== "expired") return null; + return { watchId: rest.slice(0, split), outcome }; +} + +/** The watch a wake belongs to, when the host passed its watches down. */ +export function findWakeWatch(watches: WakeWatch[] | undefined, watchId: string) { + return watches?.find((watch) => watch.id === watchId); +} + +/** + * The watch's resolution, falling back to what the transport can prove for a row + * written before the `resolution` column existed: `fired` is unambiguous, `expired` + * splits on the last check's reason. + */ +export function wakeResolution( + outcome: WakeOutcome, + watch: Pick | undefined +): WatchResolution { + if (watch?.resolution) return watch.resolution; + if (outcome === "fired") return "condition_met"; + return watch?.endedReason === "terminal_unsatisfied" + ? "condition_impossible" + : "window_completed"; +} + +/** What this banner shows, without the markup. */ +export function wakePresentation(outcome: WakeOutcome, watch: WakeWatch | undefined) { + if (!watch) return WATCH_PRESENTATION_FALLBACK; + return presentResolvedWatch({ + kind: watch.kind, + identity: watch.identity, + resolution: wakeResolution(outcome, watch), + observed: watch.observedOutcome ?? null, + }); +} + +/** + * Semantic icon to glyph. Which icon a resolved result deserves is decided in + * contracts, and the rule there is that the icon follows the observed outcome, not + * the resolution: a failed run gets `error`, not the check its `condition_met` + * would suggest. + */ +const SEMANTIC_ICON: Record JSX.Element> = { + success: CheckCircleIcon, + attention: ExclamationTriangleIcon, + error: ExclamationCircleIcon, + waiting: ClockIcon, + info: InformationCircleIcon, +}; + +const TONE_FRAME: Record = { + neutral: "border-l-border-bright bg-background-bright/40", + success: "border-l-success bg-success/10", + warning: "border-l-warning bg-warning/10", + error: "border-l-error bg-error/10", +}; + +export function WakeBanner({ + outcome, + watch, +}: { + /** The wire encoding from the wake's message id. */ + outcome: WakeOutcome; + /** The watch that woke, when the host has it. Absent: the neutral fallback. */ + watch?: WakeWatch; +}) { + const presentation = wakePresentation(outcome, watch); + const tone = presentation.tone as AgentTone; + const Icon = SEMANTIC_ICON[presentation.semanticIcon]; + const note = watchSubline(watch); + + return ( +
+ +
+

+ {presentation.label} +

+

{presentation.headline}

+ {note ?

{note}

: null} +
+
+ ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchButton.tsx b/apps/webapp/app/components/dashboard-agent/WatchButton.tsx new file mode 100644 index 00000000000..25b000cd656 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchButton.tsx @@ -0,0 +1,45 @@ +import { EyeIcon } from "@heroicons/react/20/solid"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { Button } from "~/components/primitives/Buttons"; +import { useDashboardAgent } from "./dashboardAgentLauncher"; +import { watchTooltipLabel } from "~/presenters/v3/dashboardAgent"; + +/** Posts nothing: opens the panel with the card pre-filled. Renders nothing without a provider. */ +export function WatchButton({ + spec, + label = "Watch…", + size = "small", + variant = "secondary", + fullWidth, + className, + tooltip, +}: { + spec: WatchSpec; + label?: string; + size?: "small" | "medium"; + variant?: "primary" | "secondary" | "minimal"; + fullWidth?: boolean; + className?: string; + tooltip?: string; +}) { + const agent = useDashboardAgent(); + if (!agent) { + return null; + } + + return ( + + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchCard.tsx b/apps/webapp/app/components/dashboard-agent/WatchCard.tsx new file mode 100644 index 00000000000..e395dcecf10 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchCard.tsx @@ -0,0 +1,330 @@ +/** + * The watch configuration card, opened by the Watch action. + * + * Rules it keeps: the card is ephemeral until submitted (it lives in the panel, + * not the transcript, and only a submitted outcome is persisted as a + * `watch_result` block); Customize expands in place, never a modal; in-chat + * delivery is stated as a line, so the two opt-ins stay independent checkboxes + * and never become a radio group. + * + * Pure component: draft in, markup and callbacks out. Draft rules live in + * `watch-card.ts` and wording in `app/presenters/v3/dashboardAgent`. + */ +import { EyeIcon } from "@heroicons/react/20/solid"; +import { + WATCH_WINDOW_HOURS_OPTIONS, + watchCadenceOptions, + type WatchDraft, + type WatchKind, +} from "@internal/dashboard-agent-contracts"; +import { useId, useState } from "react"; +import { Button } from "~/components/primitives/Buttons"; +import { Checkbox } from "~/components/primitives/Checkbox"; +import { Input } from "~/components/primitives/Input"; +import { AgentSpinner } from "~/components/primitives/Spinner"; +import { cn } from "~/utils/cn"; +import { ChatSystemBlock } from "./chat-layout"; +import { + variantsOf, + watchDraftError, + withAgeMinutes, + withCadence, + withFollowUp, + withThreshold, + withVariant, + withWindow, +} from "./watch-card"; +import { + formatWatchCadence, + formatWatchWindow, + WATCH_IN_CHAT_DELIVERY_LINE, + watchConditionLabel, + watchDurationLabel, + watchSubjectLabel, +} from "~/presenters/v3/dashboardAgent"; + +/** How the condition variants are named in the picker. Short, not sentences. */ +const VARIANT_LABEL: Record = { + run_start: "when it starts", + run_finished: "when it finishes", + run_failed: "if it fails", + backlog_drain: "when it drains", + queue_depth_above: "if it grows", + queue_depth_below: "when it's back below", + queue_stalled: "if it stops moving", + queue_oldest_age: "if runs wait too long", + error_recurrence: "if it recurs", + health_recovery: "when it recovers", +}; + +/** Hoisted so the submit button's icon component keeps a stable identity. */ +function ButtonSpinner() { + return ; +} + +/** Controlled, unlike `CheckboxWithLabel`: the draft is the only thing that says what's on. */ +function Toggle({ + label, + checked, + disabled, + onChange, +}: { + label: string; + checked: boolean; + disabled: boolean; + onChange: (checked: boolean) => void; +}) { + const id = useId(); + return ( +
+ onChange(event.target.checked)} + className="mt-1" + /> + +
+ ); +} + +/** One choice in an inline picker. */ +function Choice({ + selected, + disabled, + onSelect, + children, +}: { + selected: boolean; + disabled: boolean; + onSelect: () => void; + children: React.ReactNode; +}) { + return ( + + ); +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ {label} +
{children}
+
+ ); +} + +export function WatchCard({ + draft, + onChange, + onSubmit, + onCancel, + /** Start expanded: the gallery's Customize state, and a free-text pre-fill. */ + defaultExpanded = false, + /** The submit is in flight: the card stays, disabled, so nothing moves. */ + pending = false, + /** A refusal from the server (cap, duplicate, network). */ + error, +}: { + draft: WatchDraft; + onChange: (draft: WatchDraft) => void; + onSubmit: () => void; + onCancel?: () => void; + defaultExpanded?: boolean; + pending?: boolean; + error?: string | null; +}) { + const [expanded, setExpanded] = useState(defaultExpanded); + const { spec } = draft; + const variants = variantsOf(draft); + // Local validation first: a draft the schema would refuse never reaches the server. + const localError = watchDraftError(draft); + const blocked = localError !== null || pending; + + return ( + } + actions={ + <> + {/* One confirm, expanded or not: an expanded card is submitted as shown. */} + + {!expanded ? ( + + ) : null} + {onCancel ? ( + + ) : null} + + } + > +

+ Watch {watchSubjectLabel(spec)} +

+ {!expanded ? ( + <> +

{watchConditionLabel(spec)}

+

{watchDurationLabel(spec)}

+ + ) : null} +

{WATCH_IN_CHAT_DELIVERY_LINE}

+ + {expanded ? ( +
+ {/* Kinds with no second condition variant must not show an empty picker. */} + {variants.length > 1 ? ( + + {variants.map((kind) => ( + { + if (kind !== spec.kind) onChange(withVariant(draft, kind)); + }} + > + {VARIANT_LABEL[kind]} + + ))} + + ) : ( + + {watchConditionLabel(spec)} + + )} + + {/* One contextual parameter per condition, only where one exists. */} + {spec.kind === "queue_depth_above" || spec.kind === "queue_depth_below" ? ( + + + onChange(withThreshold(draft, Number.parseInt(event.target.value, 10))) + } + aria-label="Queue depth threshold" + /> + + ) : null} + + {spec.kind === "queue_oldest_age" ? ( + + + onChange(withAgeMinutes(draft, Number.parseInt(event.target.value, 10))) + } + aria-label="Wait limit in minutes" + /> + minutes + + ) : null} + + + {WATCH_WINDOW_HOURS_OPTIONS.map((hours) => ( + onChange(withWindow(draft, hours))} + > + {formatWatchWindow(hours)} + + ))} + + + {/* Cadence options come from the kind's schema limits, so an aggregate + watch can never be offered a 1-minute hot loop. */} + + {watchCadenceOptions(spec.kind).map((minutes) => ( + onChange(withCadence(draft, minutes))} + > + {formatWatchCadence(minutes)} + + ))} + + + {/* Two independent opt-ins under a fixed delivery line, never a radio + group, so "email instead of chat" is not expressible. */} + +
+ + onChange(withFollowUp(draft, { investigateOnAttention: checked })) + } + /> + onChange(withFollowUp(draft, { notifyExternally: checked }))} + /> +
+
+
+ ) : null} + + {/* Errors live and die with the card: nothing is persisted. */} + {localError || error ? ( +

{localError ?? error}

+ ) : null} +
+ ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchChips.tsx b/apps/webapp/app/components/dashboard-agent/WatchChips.tsx new file mode 100644 index 00000000000..0fbdce41bab --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchChips.tsx @@ -0,0 +1,113 @@ +import { + CheckCircleIcon, + ClockIcon, + ExclamationCircleIcon, + ExclamationTriangleIcon, + InformationCircleIcon, + NoSymbolIcon, + XMarkIcon, +} from "@heroicons/react/20/solid"; +import type { + WatchObservedOutcome, + WatchResolution, + WatchSemanticIcon, + WatchStatus, +} from "@internal/dashboard-agent-contracts"; +import { AgentSpinner } from "~/components/primitives/Spinner"; +import { SimpleTooltip } from "~/components/primitives/Tooltip"; +import { cn } from "~/utils/cn"; +import { type AgentTone, TONE_ICON_COLOR } from "./agent-badges"; +import { wakePresentation } from "./WakeBanner"; +import { watchChipLabel, watchChipTooltip } from "./watch-chips"; + +/** As the panel's loader hands it over: dates are already JSON strings. */ +export type WatchChip = { + id: string; + identity: string; + status: WatchStatus; + kind: string; + note: string; + checkEveryMinutes: number; + expiresAt: string; + endedReason?: string | null; + /** Null while active; absent on rows written before the resolution column. */ + resolution?: WatchResolution | null; + observedOutcome?: WatchObservedOutcome | null; +}; + +const SEMANTIC_ICON: Record JSX.Element> = { + success: CheckCircleIcon, + attention: ExclamationTriangleIcon, + error: ExclamationCircleIcon, + waiting: ClockIcon, + info: InformationCircleIcon, +}; + +/** + * A terminal chip wears the resolved result's icon, not its lifecycle status: a + * `run_finished` watch on a failed run resolves `condition_met`. Cancellation has none. + */ +function StatusIcon({ watch }: { watch: WatchChip }) { + if (watch.status === "active") return ; + + if (watch.status === "cancelled") { + return ; + } + + const presentation = wakePresentation(watch.status === "fired" ? "fired" : "expired", watch); + const Icon = SEMANTIC_ICON[presentation.semanticIcon]; + return ( + + ); +} + +export function WatchChips({ + watches, + onCancel, +}: { + watches: WatchChip[]; + onCancel?: (watchId: string) => void; +}) { + if (watches.length === 0) return null; + + return ( +
+ watches + {watches.map((watch) => { + const label = watchChipLabel(watch); + return ( + + + {label}} + /> + {watch.status === "active" && onCancel ? ( + onCancel(watch.id)} + className="text-text-faint transition-colors hover:text-error focus-visible:text-error focus-custom" + > + + + } + /> + ) : null} + + ); + })} +
+ ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx b/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx new file mode 100644 index 00000000000..551a9683ef5 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx @@ -0,0 +1,48 @@ +/** + * What a submitted watch card leaves in the transcript, in two flavours. + * + * A confirmation states the watch's lifetime facts and is the only transcript record + * of the request. A one-shot result means the immediate check answered outright and + * no watch was created, so no chip appears, no wake arrives and there is nothing to + * cancel. + * + * Pure component: the wording is not computed here, it was frozen into the block at + * append time by `app/presenters/v3/dashboardAgent`, so a later copy change never rewrites what + * a user was already told. + */ +import { CheckCircleIcon, EyeIcon, InformationCircleIcon } from "@heroicons/react/20/solid"; +import type { WatchResultBlock as WatchResultBlockPayload } from "@internal/dashboard-agent-contracts"; +import { ChatSystemBlock } from "./chat-layout"; +import { TONE_ICON_COLOR } from "./agent-badges"; +import { cn } from "~/utils/cn"; + +/** + * Icon and label per outcome. A confirmation is not a success (nothing has happened + * yet) so it wears the neutral eye; the check belongs to the one-shot that did + * answer the question. + */ +const OUTCOME = { + watching: { label: "Watch", Icon: EyeIcon, tone: "neutral" }, + already_true: { label: "Watch", Icon: CheckCircleIcon, tone: "success" }, + impossible: { label: "Watch", Icon: InformationCircleIcon, tone: "neutral" }, +} as const; + +export function WatchResultBlock({ block }: { block: WatchResultBlockPayload }) { + const { label, Icon, tone } = OUTCOME[block.outcome] ?? OUTCOME.watching; + + return ( + } + > +

{block.headline}

+ {block.lifetime ?

{block.lifetime}

: null} + {block.detail ?

{block.detail}

: null} + {(block.followUp ?? []).map((line) => ( +

+ {line} +

+ ))} +
+ ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx b/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx new file mode 100644 index 00000000000..8170c37b07f --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx @@ -0,0 +1,133 @@ +/** + * The dashboard-wide signal that a watch woke a chat while the panel was closed. + * + * Persistent by design: a wake answers a question asked minutes or hours ago, so it + * waits until dismissed rather than expiring on a timer. Dismissing does not mark + * the chat read (reading happens in the panel), so the launcher's dot survives a + * swatted toast. + */ +import { toast } from "sonner"; +import { Button } from "~/components/primitives/Buttons"; +import { ToastUI } from "~/components/primitives/Toast"; +import type { WatchObservedOutcome, WatchResolution } from "@internal/dashboard-agent-contracts"; +import { wakeResolution } from "./WakeBanner"; +import { presentResolvedWatch, WATCH_PRESENTATION_FALLBACK } from "~/presenters/v3/dashboardAgent"; + +/** Matches sonner's default toast width, same as the app's other toasts. */ +const TOAST_WIDTH = 356; + +/** More new wakes than this at once collapse into one summary toast. */ +export const WAKE_TOAST_MAX_INDIVIDUAL = 3; + +export type WatchWake = { + watchId: string; + chatId: string; + /** The wire encoding off the row. Not the outcome; see `resolution`. */ + outcome: "fired" | "expired"; + note: string; + /** + * What actually happened, frozen on the row by the resolving check. The toast, + * the banner and the email take their headline from the same presenter so they + * cannot disagree. Absent on a row written before the resolution model, where the + * presenter falls back rather than guessing. + */ + kind?: string; + identity?: string; + resolution?: WatchResolution | null; + observedOutcome?: WatchObservedOutcome | null; + /** Landed after the chat's read marker. The dot counts these; the toast fires either way. */ + unread?: boolean; +}; + +/** + * The toast's title: the fact, or the neutral fallback when this wake predates the + * resolution model. The wording is `app/presenters/v3/dashboardAgent`'s; this only decides + * which watch to ask it about. + */ +export function watchWakeToastTitle(wake: WatchWake): string { + if (!wake.kind || !wake.identity) return WATCH_PRESENTATION_FALLBACK.headline; + return presentResolvedWatch({ + kind: wake.kind, + identity: wake.identity, + resolution: wakeResolution(wake.outcome, { resolution: wake.resolution ?? null }), + observed: wake.observedOutcome ?? null, + }).headline; +} + +function WakeToastUI({ + t, + title, + message, + onOpenChat, +}: { + t: string; + title: string; + message: string; + onOpenChat: () => void; +}) { + return ( + { + onOpenChat(); + toast.dismiss(t); + }} + > + Open chat + + } + /> + ); +} + +function show(node: (t: string) => React.ReactElement, id: string) { + toast.custom((t) => node(t as string), { + // Manual dismissal only — see the file comment. + duration: Infinity, + // Keyed so a re-render or a duplicate poll can't stack the same wake twice. + id, + }); +} + +/** + * One persistent toast for a single wake. `onOpenChat` is given the chat the wake + * happened in, not whichever chat the panel had open last. + */ +export function showWatchWakeToast(wake: WatchWake, onOpenChat: (chatId: string) => void) { + show( + (t) => ( + onOpenChat(wake.chatId)} + /> + ), + `watch-wake-${wake.watchId}` + ); +} + +/** One persistent toast standing in for a batch too large to narrate one by one. */ +export function showWatchWakesSummaryToast(count: number, onOpenChat: () => void) { + show( + (t) => ( + + ), + // One id for all summaries: a later poll rewrites the count in place instead + // of stacking a second never-expiring toast on top of the first. + "watch-wakes-summary" + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts b/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts index 12a8cc8ff85..f58408f500d 100644 --- a/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts +++ b/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts @@ -80,6 +80,7 @@ describe("chat-layout enforcement", () => { "ChatToolRow", "ChatNote", "ChatStatusLine", + "ChatWakeSlot", "ChatActionsRow", ]) { expect(source, name).toContain(`export function ${name}(`); diff --git a/apps/webapp/app/components/dashboard-agent/chat-layout.tsx b/apps/webapp/app/components/dashboard-agent/chat-layout.tsx index 7139f24b07b..d281f64b6a5 100644 --- a/apps/webapp/app/components/dashboard-agent/chat-layout.tsx +++ b/apps/webapp/app/components/dashboard-agent/chat-layout.tsx @@ -12,6 +12,7 @@ const TURN_GAP = "space-y-4"; const TURN_BODY_GAP = "space-y-2"; const ROW_GAP = "gap-2"; const CHIP_GAP = "gap-1.5"; +const UNIT_GAP = "space-y-1.5"; const SCROLLER = "flex-1 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control"; @@ -148,6 +149,21 @@ export function ChatStatusLine({ ); } +export function ChatWakeSlot({ + banner, + children, +}: { + banner: React.ReactNode; + children: React.ReactNode; +}) { + return ( +
+ {banner} + {children} +
+ ); +} + const BLOCK_LINE_GAP = "space-y-1"; const BLOCK_INSET = "px-3 py-2.5"; diff --git a/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx b/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx index 67c018c385a..c3c34d47df7 100644 --- a/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx +++ b/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx @@ -1,3 +1,4 @@ +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; import { createContext, useContext } from "react"; import { Button } from "~/components/primitives/Buttons"; import { ShortcutKey } from "~/components/primitives/ShortcutKey"; @@ -18,6 +19,10 @@ type DashboardAgentContextValue = { setOpen: (open: boolean) => void; /** Sent as the first message of a new chat; with a chat open it only fills the composer. */ openWith: (text: string) => void; + /** Nothing is posted or persisted until the card is submitted. */ + openWithWatch: (spec: WatchSpec) => void; + /** Polled only while the panel is closed; 0 while it is open. */ + unreadWakes: number; }; const DashboardAgentContext = createContext(null); @@ -35,11 +40,13 @@ export function DashboardAgentLauncher() { return null; } - const { open, setOpen } = agent; + const { open, setOpen, unreadWakes } = agent; if (open) { return null; } + const hasUnread = unreadWakes > 0; + return ( - + {hasUnread && ( + // The ring matches the `NavBar` surface the launcher sits on. + + )} } /> diff --git a/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts b/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts index 3fd4699146d..2cae37107b9 100644 --- a/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts +++ b/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts @@ -31,7 +31,7 @@ export const demoConcurrencySaturationSignal: AgentPageSignal = { severity: "crit", }; -// Priority order. +// Priority order. `SIGNAL_PRIORITY` in the registry mirrors this. export const demoSignalsByPriority: AgentPageSignal[] = [ demoFreshFailureSignal, demoWaitingRunSignal, @@ -141,6 +141,12 @@ export const demoPromptSets: Record = { "How many other runs failed with this error in the last hour?", "contextual" ), + prompt( + "watch-retry", + "Tell me when it retries", + `Watch ${DEMO_WORLD.failedRunId} and tell me when it finishes.`, + "contextual" + ), DEFAULT_PROMPTS[1]!, ], waitingRun: [ @@ -195,6 +201,12 @@ export const demoPromptSets: Record = { "Explain this error and what usually causes it.", "promoted" ), + prompt( + "watch-recurrence", + "Tell me if it comes back", + "Watch this error and tell me if it happens again.", + "contextual" + ), DEFAULT_PROMPTS[1]!, ], queue: [ @@ -224,7 +236,7 @@ export const demoPromptSets: Record = { other: DEFAULT_PROMPTS, }; -export const demoDismissedPromptIds: string[] = []; +export const demoDismissedPromptIds: string[] = [demoId("prompt-watch-retry")]; export const demoPromptsAfterDismissal: SuggestedPrompt[] = demoPromptSets.failedRun .filter((p) => !demoDismissedPromptIds.includes(p.id)) diff --git a/apps/webapp/app/components/dashboard-agent/list-row.tsx b/apps/webapp/app/components/dashboard-agent/list-row.tsx index 6fe35f267e3..a5b80d873ff 100644 --- a/apps/webapp/app/components/dashboard-agent/list-row.tsx +++ b/apps/webapp/app/components/dashboard-agent/list-row.tsx @@ -20,6 +20,7 @@ export function AgentListRow({ meta, status, variant = "default", + unread = false, onSelect, action, }: { @@ -27,6 +28,7 @@ export function AgentListRow({ meta?: ReactNode; status?: ReactNode; variant?: AgentListRowVariant; + unread?: boolean; onSelect: () => void; /** Use {@link AgentListRowAction}. */ action?: ReactNode; @@ -38,12 +40,19 @@ export function AgentListRow({ onClick={onSelect} className={cn( "flex min-w-0 flex-1 items-center gap-2 rounded-md border px-3 py-2 text-left text-sm outline-hidden transition focus-custom", - ROW_VARIANTS[variant] + ROW_VARIANTS[variant], + unread && "text-text-bright" )} > {status ? ( {status} ) : null} + {unread ? ( + <> + + Unread. + + ) : null} {label} {meta ? {meta} : null} diff --git a/apps/webapp/app/components/dashboard-agent/message-quota.ts b/apps/webapp/app/components/dashboard-agent/message-quota.ts index 21edb9ab9fb..f65481c8705 100644 --- a/apps/webapp/app/components/dashboard-agent/message-quota.ts +++ b/apps/webapp/app/components/dashboard-agent/message-quota.ts @@ -1,3 +1,5 @@ +import { isWatchRequestMessageId } from "@internal/dashboard-agent-contracts"; + // Counted per user across their chats in the org, not per chat, which "New chat" // would reset. export const FREE_PLAN_MESSAGE_LIMIT = 20; @@ -25,6 +27,12 @@ export function resolveMessageQuota({ : { kind: "within", used, limit, remaining }; } -export function countUserMessages(messages: { role: string }[]): number { - return messages.reduce((total, message) => (message.role === "user" ? total + 1 : total), 0); +// A watch's consent record is a user message the person never typed, so it is +// excluded here exactly as the stored count excludes it. +export function countUserMessages(messages: { role: string; id?: string }[]): number { + return messages.reduce( + (total, message) => + message.role === "user" && !isWatchRequestMessageId(message.id) ? total + 1 : total, + 0 + ); } diff --git a/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts b/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts index 6cbc02d3573..2ff0d61403f 100644 --- a/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts +++ b/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { pendingNavigateIntents } from "./pending-intents"; +import { pendingNavigateIntents, pendingWatchIntents } from "./pending-intents"; describe("pendingNavigateIntents", () => { const uri = "trigger://proj_abc/env_123/run/run_abc"; @@ -46,3 +46,75 @@ describe("pendingNavigateIntents", () => { ).toEqual([{ kind: "navigate", target: uri }]); }); }); + +describe("pendingWatchIntents", () => { + const spec = { + kind: "run_finished", + runId: "run_abc", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when the receipt run finishes", + }; + const toolPart = (toolCallId: string, state = "output-available") => ({ + type: "tool-schedule_watch", + state, + toolCallId, + output: { intent: { kind: "watch", spec } }, + }); + + it("returns the proposed spec from a completed schedule_watch call, once", () => { + const seen = new Set(); + const messages = [{ id: "m1", parts: [toolPart("call-1")] }]; + + expect(pendingWatchIntents(messages, seen)).toEqual([{ kind: "watch", spec }]); + expect(pendingWatchIntents(messages, seen)).toEqual([]); + }); + + it("ignores a call still running, and a spec the contract rejects", () => { + expect( + pendingWatchIntents([{ id: "m1", parts: [toolPart("call-1", "input-available")] }], new Set()) + ).toEqual([]); + + const invalid = [ + { + id: "m1", + parts: [ + { + ...toolPart("call-2"), + output: { intent: { kind: "watch", spec: { kind: "run_finished" } } }, + }, + ], + }, + ]; + expect(pendingWatchIntents(invalid, new Set())).toEqual([]); + }); + + it("never reopens a proposal seeded from loaded history", () => { + const history = [{ id: "m1", parts: [toolPart("call-1")] }]; + const seen = new Set(); + pendingWatchIntents(history, seen); + + expect(pendingWatchIntents(history, seen)).toEqual([]); + expect( + pendingWatchIntents([...history, { id: "m2", parts: [toolPart("call-2")] }], seen) + ).toEqual([{ kind: "watch", spec }]); + }); + + it("doesn't confuse a navigate result for a watch", () => { + const messages = [ + { + id: "m1", + parts: [ + { + type: "tool-navigate_to", + state: "output-available", + toolCallId: "call-1", + output: { intent: { kind: "navigate", target: "trigger://p/e/run/run_abc" } }, + }, + ], + }, + ]; + + expect(pendingWatchIntents(messages, new Set())).toEqual([]); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/pending-intents.ts b/apps/webapp/app/components/dashboard-agent/pending-intents.ts index 6b58ba04d31..93a1d790656 100644 --- a/apps/webapp/app/components/dashboard-agent/pending-intents.ts +++ b/apps/webapp/app/components/dashboard-agent/pending-intents.ts @@ -40,3 +40,11 @@ export function pendingNavigateIntents( ): Array> { return pendingToolIntents(messages, seen, "tool-navigate_to", "navigate"); } + +// `schedule_watch` only proposes: the panel creates the watch. +export function pendingWatchIntents( + messages: ReadonlyArray, + seen: Set +): Array> { + return pendingToolIntents(messages, seen, "tool-schedule_watch", "watch"); +} diff --git a/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx b/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx index c4c8f248133..add6fb256ff 100644 --- a/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx +++ b/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx @@ -310,6 +310,13 @@ export function ReportNoteBlock({ label, children }: { label: string; children: // and `note` is prose for an option stated rather than offered. export { reportFooterStyle, type ReportFooterStyle }; +/** + * The recovery-watch offer. No report emits it; the card adds it. Two codes + * because it is phrased differently when it is the only thing on offer. + */ +export const FOOTER_WATCH_CODE = "watch_recovery"; +export const FOOTER_WATCH_ONLY_CODE = "watch_recovery_only"; + /** A dimmed line that accompanies a row entry. */ const FOOTER_NOTE_LINES: Record = { check_control_plane: "There's nothing to fix on your side.", diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts index 8f594a1183c..e5651a95d14 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts @@ -97,6 +97,11 @@ export function pageSlotPrompts(page: AgentPage): PageSlotPrompts { "Why does this keep happening?", "Investigate this error — why does it keep coming back, and which runs are affected?" ), + watch: def( + "error-watch-recurrence", + "Tell me if it comes back", + "Watch this error and tell me if it happens again." + ), explain: def( "error-similar", "Find similar failures", @@ -125,6 +130,11 @@ export function pageSlotPrompts(page: AgentPage): PageSlotPrompts { queueBacklogPrompt(page.name) ) : undefined, + watch: def( + "queue-watch-drain", + "Tell me when the backlog drains", + `Watch the ${page.name} queue and tell me when the backlog drains.` + ), status: def( "queue-backlog", "How big is the backlog?", diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts index aae8e0e4cf0..5e5f7e5690a 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts @@ -25,12 +25,13 @@ export const ctx = (id: string, label: string, prompt: string) => make(id, label, prompt, "contextual"); /** The slots after the promoted one, in display order. */ -export const PROMPT_SLOTS = ["investigate", "status", "explain", "docs"] as const; +export const PROMPT_SLOTS = ["investigate", "watch", "status", "explain", "docs"] as const; export type PromptSlot = (typeof PROMPT_SLOTS)[number]; export type PageSlotPrompts = { investigate?: SuggestedPrompt; + watch?: SuggestedPrompt; status?: SuggestedPrompt; explain: SuggestedPrompt; docs: SuggestedPrompt; diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts index 80ce032b0bf..ed523193fd8 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts @@ -25,27 +25,33 @@ const docsId = (key: keyof typeof demoPageContexts) => pageSlotPrompts(demoPageContexts[key].page).docs.id; describe("resolveSuggestedPrompts", () => { - it("fills every slot the page has, given a promoted chip, signals and defaults", () => { + it("fills all five slots when the page has a promoted chip, signals and defaults", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.error, { promoted, now: NOW }); - expect(prompts).toHaveLength(4); + expect(prompts).toHaveLength(5); expect(ids(prompts)).toEqual([ promoted.id, "sp:fresh-failure", + "sp:error-watch-recurrence", "sp:error-similar", docsId("error"), ]); expect(prompts[0]?.source).toBe("promoted"); }); - it("drops a slot when nothing is promoted", () => { + it("drops to four slots when nothing is promoted", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW }); - expect(prompts).toHaveLength(3); - expect(ids(prompts)).toEqual(["sp:fresh-failure", "sp:error-similar", docsId("error")]); + expect(prompts).toHaveLength(4); + expect(ids(prompts)).toEqual([ + "sp:fresh-failure", + "sp:error-watch-recurrence", + "sp:error-similar", + docsId("error"), + ]); }); - it("shows explain + docs only when no investigate applies", () => { + it("shows explain + docs only when no investigate or watch applies", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.deployment, { now: NOW }); expect(ids(prompts)).toEqual(ids(pageDefaultPrompts(demoPageContexts.deployment.page))); @@ -73,7 +79,7 @@ describe("resolveSuggestedPrompts", () => { } }); - it("orders promoted, then investigate, then status, then explain", () => { + it("orders promoted, then investigate, then watch, then explain", () => { const context = { ...demoPageContexts.queue, signals: [...demoPageContexts.queue.signals, demoFreshFailureSignal], @@ -84,7 +90,8 @@ describe("resolveSuggestedPrompts", () => { expect(ids(prompts)).toEqual([ promoted.id, "sp:fresh-failure", - "sp:queue-backlog", + // waiting_run beats concurrency_saturation for the watch slot. + "sp:waiting-run", "sp:queue-state", docsId("queue"), ]); @@ -122,10 +129,10 @@ describe("resolveSuggestedPrompts", () => { const full = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW }); const dismissed = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW, - dismissedIds: ["sp:error-similar"], + dismissedIds: ["sp:error-watch-recurrence"], }); - expect(ids(dismissed)).not.toContain("sp:error-similar"); + expect(ids(dismissed)).not.toContain("sp:error-watch-recurrence"); expect(dismissed).toHaveLength(full.length - 1); expect(dismissed.at(-1)?.id).toBe(docsId("error")); }); @@ -161,7 +168,12 @@ describe("resolveSuggestedPrompts", () => { expect(failure?.prompt).toContain("12m ago"); }); - it("words the slow-run chip for its slot", () => { + it("words the waiting-run and slow-run chips for their slots", () => { + const waiting = resolveSuggestedPrompts(demoPageContexts.waitingRun, { now: NOW }); + const waitingChip = waiting.find((p) => p.id === "sp:waiting-run"); + expect(waitingChip?.label).toBe("Tell me when this run starts"); + expect(waitingChip?.prompt).toContain("queue"); + const slow = resolveSuggestedPrompts(demoPageContexts.slowRun, { now: NOW }); expect(slow[0]?.label).toBe("~7.8x slower than usual"); }); diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts index 9264ae78690..b8575acd459 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts @@ -68,7 +68,7 @@ export function resolveSuggestedPromptsBySlot( } // Over the cap, optional slots yield in this order; promoted, explain and docs never yield. - const yieldOrder: ResolvedPromptSlot[] = ["status", "investigate"]; + const yieldOrder: ResolvedPromptSlot[] = ["status", "watch", "investigate"]; let trimmed = resolved; for (const slot of yieldOrder) { if (trimmed.length <= SUGGESTED_PROMPT_CAP) break; diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts index d5faf17653f..6c4a2f74982 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts @@ -10,14 +10,20 @@ import type { } from "@internal/dashboard-agent-contracts"; import { ctx, type PromptSlot } from "./prompt-chips"; -/** A kind with no entry produces no chip. */ -export const SIGNAL_SLOT: Partial> = { +export const SIGNAL_SLOT: Record = { fresh_failure: "investigate", slow_run: "investigate", + waiting_run: "watch", + concurrency_saturation: "watch", }; -/** Signal precedence within a slot. */ -export const SIGNAL_PRIORITY: AgentPageSignalKind[] = ["fresh_failure", "slow_run"]; +/** Signal precedence within a slot. Mirrors `demoSignalsByPriority` in the fixtures. */ +export const SIGNAL_PRIORITY: AgentPageSignalKind[] = [ + "fresh_failure", + "waiting_run", + "slow_run", + "concurrency_saturation", +]; /** "3m", "2h", "4d". */ export function formatAgo(ms: number): string { @@ -47,6 +53,15 @@ export function promptForSignal(signal: AgentPageSignal, now: number): Suggested ); } + case "waiting_run": + return ctx( + "waiting-run", + "Tell me when this run starts", + signal.queue + ? `Watch ${signal.runId} and tell me when it leaves the ${signal.queue} queue.` + : `Watch ${signal.runId} and tell me when it starts running.` + ); + case "slow_run": { if (signal.baselineP95Ms <= 0) return undefined; const factor = formatMultiplier(signal.durationMs / signal.baselineP95Ms); @@ -56,6 +71,13 @@ export function promptForSignal(signal: AgentPageSignal, now: number): Suggested `${signal.runId} is running ~${factor} slower than this task's usual p95. Investigate why.` ); } + + case "concurrency_saturation": + return ctx( + "concurrency-saturation", + "Tell me when the backlog drains", + "Concurrency is saturated right now. Watch it and tell me when the backlog drains." + ); } } @@ -79,18 +101,17 @@ export function contextualPromptsBySlot( ): Record { const bySlot: Record = { investigate: [], + watch: [], status: [], explain: [], docs: [], }; for (const kind of SIGNAL_PRIORITY) { - const slot = SIGNAL_SLOT[kind]; - if (!slot) continue; for (const signal of context.signals) { if (signal.kind !== kind) continue; const prompt = promptForSignal(signal, now); - if (prompt) bySlot[slot].push(prompt); + if (prompt) bySlot[SIGNAL_SLOT[kind]].push(prompt); } } diff --git a/apps/webapp/app/components/dashboard-agent/tool-labels.ts b/apps/webapp/app/components/dashboard-agent/tool-labels.ts index d3bd07f52d4..4972cdb2d5b 100644 --- a/apps/webapp/app/components/dashboard-agent/tool-labels.ts +++ b/apps/webapp/app/components/dashboard-agent/tool-labels.ts @@ -21,6 +21,7 @@ const TOOL_LABELS: Record = { search_docs: "Searching the docs", get_current_page: "Reading the current page", navigate_to: "Opening the page", + schedule_watch: "Filling in a watch", list_alerts: "Listing alerts", create_alert: "Creating an alert", delete_alert: "Deleting an alert", diff --git a/apps/webapp/app/components/dashboard-agent/turn-error.test.ts b/apps/webapp/app/components/dashboard-agent/turn-error.test.ts index 9975bdd1f49..4bf646e4fbb 100644 --- a/apps/webapp/app/components/dashboard-agent/turn-error.test.ts +++ b/apps/webapp/app/components/dashboard-agent/turn-error.test.ts @@ -7,7 +7,7 @@ const failure = { id: "turn-error:0" }; describe("the failed-turn record", () => { it("recognises the agent's failure message id", () => { expect(isTurnErrorMessageId("turn-error:3")).toBe(true); - expect(isTurnErrorMessageId("msg_1")).toBe(false); + expect(isTurnErrorMessageId("wake:watch:watch_1:fired")).toBe(false); expect(isTurnErrorMessageId(undefined)).toBe(false); }); diff --git a/apps/webapp/app/components/dashboard-agent/turn-error.ts b/apps/webapp/app/components/dashboard-agent/turn-error.ts index 1fecf35c1ba..0a451401088 100644 --- a/apps/webapp/app/components/dashboard-agent/turn-error.ts +++ b/apps/webapp/app/components/dashboard-agent/turn-error.ts @@ -1,7 +1,8 @@ /** * A failed turn is recorded in the transcript by the agent, under the message id - * `turn-error:{turn}`. The prefix is the transport convention, recognised here so - * the panel can tell a stored failure record apart from an ordinary answer. + * `turn-error:{turn}`. Same arrangement as a wake's `wake:watch:…` id: the prefix + * is the transport convention, recognised here so the panel can tell a stored + * failure record apart from an ordinary answer. * * Live, a failure arrives as the stream's error chunk and `useChat` surfaces it as * the retry callout. The stored record is what a reload reads. Both must never show diff --git a/apps/webapp/app/components/dashboard-agent/view-actions.test.ts b/apps/webapp/app/components/dashboard-agent/view-actions.test.ts index e156c74beaa..e46619ce228 100644 --- a/apps/webapp/app/components/dashboard-agent/view-actions.test.ts +++ b/apps/webapp/app/components/dashboard-agent/view-actions.test.ts @@ -3,6 +3,20 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { renderableActions } from "./view-actions"; +const watchAction: ActionsBlockAction = { + label: "Set up a watch", + intent: { + kind: "watch", + spec: { + kind: "error_recurrence", + fingerprint: "a1b2c3", + checkEveryMinutes: 15, + maxHours: 6, + note: "the TypeError in send-order-receipt", + }, + }, +}; + const askAction: ActionsBlockAction = { label: "Investigate it", intent: { kind: "ask", prompt: "Investigate the send-order-receipt failures." }, @@ -25,6 +39,10 @@ describe("renderableActions", () => { expect(renderableActions([navigate])).toEqual([navigate]); }); + it("keeps a watch action, spec intact — that spec is what pre-fills the card", () => { + expect(renderableActions([watchAction, askAction])).toEqual([watchAction, askAction]); + }); + it("can filter every action out, leaving nothing to render", () => { expect( renderableActions([{ label: "Nowhere", intent: { kind: "navigate", target: "nope" } }]) diff --git a/apps/webapp/app/components/dashboard-agent/view-catalog.tsx b/apps/webapp/app/components/dashboard-agent/view-catalog.tsx index 24f719a5219..26389d80908 100644 --- a/apps/webapp/app/components/dashboard-agent/view-catalog.tsx +++ b/apps/webapp/app/components/dashboard-agent/view-catalog.tsx @@ -5,6 +5,7 @@ import { InvestigationCard } from "./InvestigationCard"; import { ReportView, type ResolvedUri } from "./ReportView"; import { RunDiagnosisCard } from "./RunDiagnosisCard"; import { blockKey, latestRevisionBlocks } from "./view-blocks"; +import { WatchResultBlock } from "./WatchResultBlock"; // Unknown block types are skipped, so an older or newer agent cannot render // arbitrary content. A new block needs a `case` here and a `viewBlockSchema` member. @@ -43,6 +44,9 @@ export function ViewBlocks({ onIntent={onIntent} /> ); + // Host-emitted only, so the model cannot fabricate a confirmation. + case "watch_result": + return ; case "report": return ( { + it("still reads the as-built two-value wake id", () => { + expect(wakeRefFromMessageId("wake:watch:watch_1:fired")).toEqual({ + watchId: "watch_1", + outcome: "fired", + }); + expect(wakeRefFromMessageId("wake:watch:watch_1:expired")).toEqual({ + watchId: "watch_1", + outcome: "expired", + }); + expect(wakeRefFromMessageId("msg_1")).toBeNull(); + }); +}); + +describe("wakeResolution", () => { + it("prefers the row's resolution", () => { + expect(wakeResolution("expired", { resolution: "condition_impossible" })).toBe( + "condition_impossible" + ); + }); + + it("reconstructs one for a row written before the resolution column", () => { + expect(wakeResolution("fired", { endedReason: null })).toBe("condition_met"); + expect(wakeResolution("expired", { endedReason: "terminal_unsatisfied" })).toBe( + "condition_impossible" + ); + expect(wakeResolution("expired", { endedReason: "not_met_by_expiry" })).toBe( + "window_completed" + ); + expect(wakeResolution("expired", undefined)).toBe("window_completed"); + }); +}); + +describe("wakePresentation", () => { + it("states the fact, not a generic watch update", () => { + const presented = wakePresentation("fired", { + ...runWatch, + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_SUCCESSFULLY", + durationMs: 4200, + }, + }); + expect(presented.headline).toBe("Run run_abc123 finished"); + expect(presented.label).toBe("Watch update"); + expect(presented.category).toBe("positive"); + }); + + it("shows a failed run as a failure, on the same resolution", () => { + const presented = wakePresentation("fired", { + ...runWatch, + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: null, + }, + }); + expect(presented.headline).toBe("Run run_abc123 failed"); + expect(presented.category).toBe("attention"); + expect(presented.semanticIcon).not.toBe("success"); + }); + + it("names the queue in a drain headline", () => { + expect( + wakePresentation("fired", { + id: "watch_2", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "condition_met", + }).headline + ).toBe("email-sends queue drained"); + }); + + it("reports the threshold watch with its number", () => { + expect( + wakePresentation("fired", { + id: "watch_3", + kind: "queue_depth_above", + identity: "queue_depth_above:email-sends:500", + note: "", + resolution: "condition_met", + observedOutcome: { + kind: "queue_depth_above", + verified: true, + depth: 612, + threshold: 500, + }, + }).headline + ).toBe("email-sends queue is still above 500"); + }); + + it("treats a completed window as an answer, not silence", () => { + const presented = wakePresentation("expired", { + id: "watch_4", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "window_completed", + observedOutcome: { kind: "backlog_drain", verified: true, depth: 42 }, + }); + expect(presented.headline).toBe("email-sends queue is still at 42"); + expect(presented.category).toBe("attention"); + }); + + it("says the condition couldn't be confirmed when the final read failed", () => { + expect( + wakePresentation("expired", { + id: "watch_5", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "window_completed", + observedOutcome: { kind: "backlog_drain", verified: false, depth: null }, + }).headline + ).toBe("The watch ended without a confirmed answer"); + }); + + it("falls back without guessing an outcome when the watch is gone", () => { + const presented = wakePresentation("fired", undefined); + expect(presented.headline).toBe("The watch woke this chat up on its own."); + expect(presented.category).toBe("neutral"); + }); + + it("says an error recurred, and that a quiet window was good news", () => { + const error = { + id: "watch_6", + kind: "error_recurrence", + identity: "error_recurrence:a1b2c3d4e5f6", + note: "", + }; + expect(wakePresentation("fired", { ...error, resolution: "condition_met" })).toMatchObject({ + headline: "Error a1b2c3d4 happened again", + category: "attention", + }); + expect(wakePresentation("expired", { ...error, resolution: "window_completed" })).toMatchObject( + { headline: "Error a1b2c3d4 stayed quiet", category: "positive" } + ); + }); + + it("says a queue came back below its threshold, and when it never did", () => { + const below = { + id: "watch_below", + kind: "queue_depth_below", + identity: "queue_depth_below:email-sends:100", + note: "", + }; + expect( + wakePresentation("fired", { + ...below, + resolution: "condition_met", + observedOutcome: { kind: "queue_depth_below", verified: true, depth: 42, threshold: 100 }, + }) + ).toMatchObject({ headline: "email-sends queue is back below 100", category: "positive" }); + + expect( + wakePresentation("expired", { + ...below, + resolution: "window_completed", + observedOutcome: { kind: "queue_depth_below", verified: true, depth: 780, threshold: 100 }, + }) + ).toMatchObject({ headline: "email-sends queue is still above 100", category: "attention" }); + }); + + it("says a queue is stuck at the depth it stalled on, and that it kept moving", () => { + const stalled = { + id: "watch_stalled", + kind: "queue_stalled", + identity: "queue_stalled:email-sends", + note: "", + }; + expect( + wakePresentation("fired", { + ...stalled, + resolution: "condition_met", + observedOutcome: { + kind: "queue_stalled", + verified: true, + depth: 42, + notDecreasingStreak: 3, + ticks: 3, + }, + }) + ).toMatchObject({ headline: "email-sends queue is stuck at 42", category: "attention" }); + + expect( + wakePresentation("expired", { + ...stalled, + resolution: "window_completed", + observedOutcome: { + kind: "queue_stalled", + verified: true, + depth: 3, + notDecreasingStreak: 1, + ticks: 3, + }, + }) + ).toMatchObject({ headline: "email-sends queue kept moving", category: "positive" }); + }); + + it("states the wait and the limit it passed, in minutes", () => { + const age = { + id: "watch_age", + kind: "queue_oldest_age", + identity: "queue_oldest_age:email-sends:5", + note: "", + }; + expect( + wakePresentation("fired", { + ...age, + resolution: "condition_met", + observedOutcome: { + kind: "queue_oldest_age", + verified: true, + ageMs: 12 * 60_000, + thresholdMinutes: 5, + }, + }) + ).toMatchObject({ + headline: "runs in email-sends are waiting 12m (over your 5m limit)", + category: "attention", + }); + + expect( + wakePresentation("expired", { + ...age, + resolution: "window_completed", + observedOutcome: { + kind: "queue_oldest_age", + verified: true, + ageMs: 30_000, + thresholdMinutes: 5, + }, + }) + ).toMatchObject({ headline: "email-sends queue stayed under 5m", category: "positive" }); + }); + + it("names the queue, not the threshold, when a queue-pack watch's queue is gone", () => { + for (const [kind, identity] of [ + ["queue_depth_below", "queue_depth_below:email-sends:100"], + ["queue_stalled", "queue_stalled:email-sends"], + ["queue_oldest_age", "queue_oldest_age:email-sends:5"], + ] as const) { + expect( + wakePresentation("expired", { + id: `watch_${kind}`, + kind, + identity, + note: "", + resolution: "condition_impossible", + }) + ).toMatchObject({ headline: "email-sends queue no longer exists", category: "neutral" }); + } + }); + + it("recovers health without naming an identity", () => { + expect( + wakePresentation("fired", { + id: "watch_7", + kind: "health_recovery", + identity: "health_recovery:health", + note: "", + resolution: "condition_met", + }).headline + ).toBe("Health recovered"); + }); +}); + +describe("watchWakeToastTitle", () => { + const wake = { + watchId: "watch_1", + chatId: "chat_1", + note: "tell me when the nightly invoice run finishes", + }; + + it("leads with the fact, not the notification", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "fired", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + resolution: "condition_met", + }) + ).toBe("email-sends queue drained"); + }); + + it("follows the observed outcome, so a failed run is never good news", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "fired", + kind: "run_finished", + identity: "run_finished:run_abc123", + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 1200, + }, + }) + ).toBe("Run run_abc123 failed"); + }); + + it("reconstructs a resolution for a row written before the model existed", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "expired", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + }) + ).toBe("email-sends queue still hasn't drained"); + }); + + it("claims nothing when the wake carries no watch at all", () => { + expect(watchWakeToastTitle({ ...wake, outcome: "fired" })).toBe( + "The watch woke this chat up on its own." + ); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts b/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts new file mode 100644 index 00000000000..5d38eb98425 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts @@ -0,0 +1,100 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { startWakePolling, UNREAD_POLL_INTERVAL_MS } from "./wake-poll"; + +function harness() { + let hidden = false; + const listeners = new Set<() => void>(); + const loads: number[] = []; + + const stop = startWakePolling({ + load: async () => { + loads.push(Date.now()); + }, + isHidden: () => hidden, + onVisibilityChange: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + // No jitter, so every delay is exactly one interval. + random: () => 0, + setTimer: (callback, ms) => setTimeout(callback, ms) as unknown as number, + clearTimer: (handle) => clearTimeout(handle as unknown as NodeJS.Timeout), + }); + + return { + loads, + stop, + setHidden(next: boolean) { + hidden = next; + for (const listener of listeners) listener(); + }, + listenerCount: () => listeners.size, + }; +} + +describe("startWakePolling", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it("polls once immediately and then once per interval", async () => { + const poll = harness(); + + expect(poll.loads).toHaveLength(1); + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 3); + expect(poll.loads).toHaveLength(4); + + poll.stop(); + }); + + it("asks nothing while hidden and catches up once when visible again", async () => { + const poll = harness(); + poll.setHidden(true); + + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 3); + expect(poll.loads).toHaveLength(1); + + poll.setHidden(false); + await vi.advanceTimersByTimeAsync(0); + expect(poll.loads).toHaveLength(2); + + poll.stop(); + }); + + it("keeps exactly one chain across ten rapid hide/show cycles", async () => { + const poll = harness(); + + for (let cycle = 0; cycle < 10; cycle++) { + poll.setHidden(true); + await vi.advanceTimersByTimeAsync(10); + poll.setHidden(false); + await vi.advanceTimersByTimeAsync(10); + } + + const afterCycles = poll.loads.length; + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 10); + + // One poll per interval, not ten: the resumes replaced the chain instead of + // forking it. + expect(poll.loads.length - afterCycles).toBe(10); + + poll.stop(); + }); + + it("stops every timer and listener on unmount", async () => { + const poll = harness(); + poll.setHidden(true); + poll.setHidden(false); + + poll.stop(); + const settled = poll.loads.length; + + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 10); + expect(poll.loads).toHaveLength(settled); + expect(poll.listenerCount()).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/wake-poll.ts b/apps/webapp/app/components/dashboard-agent/wake-poll.ts new file mode 100644 index 00000000000..c44b252f8f4 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/wake-poll.ts @@ -0,0 +1,68 @@ +/** + * The wake feed's poll: one self-scheduling chain per mount. A hidden tab asks nothing, a + * resume catches up once, and neither can fork the chain into a second one. + */ + +export const UNREAD_POLL_INTERVAL_MS = 60_000; + +// Added to each delay so open tabs never settle into polling on the same second. +export const UNREAD_POLL_JITTER_MS = 15_000; + +export type WakePollOptions = { + load: () => Promise; + isHidden: () => boolean; + /** Subscribe to visibility changes; returns its own unsubscribe. */ + onVisibilityChange: (listener: () => void) => () => void; + /** Seams so a test can drive the chain without real timers. */ + random?: () => number; + setTimer?: (callback: () => void, delayMs: number) => number; + clearTimer?: (handle: number) => void; +}; + +/** Start polling. The returned function stops the chain for good. */ +export function startWakePolling(options: WakePollOptions): () => void { + const random = options.random ?? Math.random; + const setTimer = options.setTimer ?? ((callback, ms) => window.setTimeout(callback, ms)); + const clearTimer = options.clearTimer ?? ((handle) => window.clearTimeout(handle)); + + let stopped = false; + let timer: number | undefined; + let loading = false; + // Each tick carries the chain it belongs to, so an orphaned callback returns + // instead of scheduling itself again. + let chain = 0; + + const tick = (generation: number) => { + if (stopped || generation !== chain) return; + + // Scheduled before the load, so a slow response can't stall the chain. + timer = setTimer( + () => tick(generation), + UNREAD_POLL_INTERVAL_MS + random() * UNREAD_POLL_JITTER_MS + ); + + if (loading || options.isHidden()) return; + loading = true; + const done = () => { + loading = false; + }; + options.load().then(done, done); + }; + + const unsubscribe = options.onVisibilityChange(() => { + if (stopped || options.isHidden()) return; + // One catch-up fetch on a new chain, replacing the pending timer rather than + // adding a second chain. + if (timer !== undefined) clearTimer(timer); + chain += 1; + tick(chain); + }); + + tick(chain); + + return () => { + stopped = true; + if (timer !== undefined) clearTimer(timer); + unsubscribe(); + }; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts b/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts new file mode 100644 index 00000000000..8b25448e318 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts @@ -0,0 +1,136 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type StorageListener = (event: { key: string | null }) => void; + +const store = new Map(); +const storageListeners = new Set(); + +// A minimal `window`: these tests run without a DOM. +const windowStub = { + localStorage: { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => void store.set(key, value), + }, + addEventListener: (_type: string, listener: StorageListener) => + void storageListeners.add(listener), + removeEventListener: (_type: string, listener: StorageListener) => + void storageListeners.delete(listener), +}; + +const { + forgetWatchActivity, + hasWatchActivity, + rememberWatchActivity, + shouldPollWakeFeed, + subscribeWatchActivity, +} = await import("./watch-activity"); + +/** What another tab writing the key looks like here. */ +function otherTabWrote(organizationId: string) { + store.set("tdev:dashboard-agent:watching", JSON.stringify([organizationId])); + for (const listener of storageListeners) listener({ key: "tdev:dashboard-agent:watching" }); +} + +describe("watch activity", () => { + beforeEach(() => { + store.clear(); + storageListeners.clear(); + vi.stubGlobal("window", windowStub); + }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("knows nothing until a watch shows up", () => { + expect(hasWatchActivity("org_1")).toBe(false); + + rememberWatchActivity("org_1"); + expect(hasWatchActivity("org_1")).toBe(true); + expect(hasWatchActivity("org_2")).toBe(false); + }); + + it("survives a reload", () => { + rememberWatchActivity("org_1"); + storageListeners.clear(); + + expect(hasWatchActivity("org_1")).toBe(true); + }); + + it("tells a tab that was already open", () => { + const woken: number[] = []; + const unsubscribe = subscribeWatchActivity(() => woken.push(1)); + + rememberWatchActivity("org_1"); + expect(woken).toHaveLength(1); + + unsubscribe(); + rememberWatchActivity("org_2"); + expect(woken).toHaveLength(1); + }); + + it("tells a tab about a watch another tab created", () => { + const woken: number[] = []; + const unsubscribe = subscribeWatchActivity(() => woken.push(1)); + + otherTabWrote("org_1"); + + expect(woken).toHaveLength(1); + expect(hasWatchActivity("org_1")).toBe(true); + unsubscribe(); + }); + + it("forgets one organization without forgetting the others", () => { + rememberWatchActivity("org_1"); + rememberWatchActivity("org_2"); + + forgetWatchActivity("org_1"); + + expect(hasWatchActivity("org_1")).toBe(false); + expect(hasWatchActivity("org_2")).toBe(true); + }); + + it("remembers at most ten organizations", () => { + for (let index = 0; index < 12; index++) rememberWatchActivity(`org_${index}`); + + expect(hasWatchActivity("org_0")).toBe(false); + expect(hasWatchActivity("org_11")).toBe(true); + }); + + describe("shouldPollWakeFeed", () => { + it("polls in a fresh browser the page load says has an unread wake", () => { + expect(shouldPollWakeFeed({ serverUnreadWakes: 1, organizationId: "org_1" })).toBe(true); + }); + + it("stays quiet when neither the page load nor this browser knows of anything", () => { + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_1" })).toBe(false); + }); + + it("polls in a fresh browser whose only signal is an active watch", () => { + // Created on another machine, nothing woken yet, no local marker. + expect( + shouldPollWakeFeed({ + serverUnreadWakes: 0, + serverHasActiveWatches: true, + organizationId: "org_1", + }) + ).toBe(true); + }); + + it("stays quiet in a fresh browser with no wake and no active watch", () => { + expect( + shouldPollWakeFeed({ + serverUnreadWakes: 0, + serverHasActiveWatches: false, + organizationId: "org_1", + }) + ).toBe(false); + }); + + it("polls without a reload once this browser sees a watch", () => { + rememberWatchActivity("org_1"); + + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_1" })).toBe(true); + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_2" })).toBe(false); + }); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-activity.ts b/apps/webapp/app/components/dashboard-agent/watch-activity.ts new file mode 100644 index 00000000000..5fa40e7892a --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-activity.ts @@ -0,0 +1,88 @@ +/** + * Which organizations this browser has seen agent watches in. This is an accelerator, not the + * gate: a watch created in this tab starts the poll without a reload. The ungated signals are the + * unread count and the active-watch flag the page load carries — see {@link shouldPollWakeFeed}. + * Shared through `localStorage`, so a watch created in one tab wakes the others. + */ + +const STORAGE_KEY = "tdev:dashboard-agent:watching"; + +// Newest ids only, so the key can't grow unbounded. +const MAX_REMEMBERED = 10; + +const listeners = new Set<() => void>(); + +function read(): string[] { + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + return raw ? (JSON.parse(raw) as string[]) : []; + } catch { + // Storage unavailable; treated as "nothing known yet". + return []; + } +} + +export function hasWatchActivity(organizationId: string): boolean { + if (typeof window === "undefined") return false; + return read().includes(organizationId); +} + +/** Called whenever a watch shows up for this org: the poll starts from here. */ +export function rememberWatchActivity(organizationId: string): void { + if (typeof window === "undefined" || hasWatchActivity(organizationId)) return; + try { + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify([...read(), organizationId].slice(-MAX_REMEMBERED)) + ); + } catch { + // Same as the read. This tab still starts polling for the rest of the session. + } + for (const listener of listeners) listener(); +} + +/** + * Called when nothing is left to be woken about. The current tab keeps polling for the rest of + * the session; the next reload starts quiet. + */ +export function forgetWatchActivity(organizationId: string): void { + if (typeof window === "undefined" || !hasWatchActivity(organizationId)) return; + try { + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify(read().filter((id) => id !== organizationId)) + ); + } catch { + // Same as the read. + } +} + +/** + * Whether this browser should poll the wake feed. Both server signals come from the page load, + * so a fresh browser polls without ever opening the panel: `serverUnreadWakes` for a wake that + * already landed, `serverHasActiveWatches` for one created elsewhere that hasn't fired yet. + */ +export function shouldPollWakeFeed(params: { + serverUnreadWakes: number; + serverHasActiveWatches?: boolean; + organizationId: string; +}): boolean { + return ( + params.serverUnreadWakes > 0 || + params.serverHasActiveWatches === true || + hasWatchActivity(params.organizationId) + ); +} + +/** Fires when this browser learns of a watch, in this tab or — via `storage` — in another one. */ +export function subscribeWatchActivity(listener: () => void): () => void { + listeners.add(listener); + const onStorage = (event: StorageEvent) => { + if (event.key === null || event.key === STORAGE_KEY) listener(); + }; + window.addEventListener("storage", onStorage); + return () => { + listeners.delete(listener); + window.removeEventListener("storage", onStorage); + }; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-card.test.ts b/apps/webapp/app/components/dashboard-agent/watch-card.test.ts new file mode 100644 index 00000000000..7cb6bb15e85 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card.test.ts @@ -0,0 +1,375 @@ +import { describe, expect, it } from "vitest"; +import { + WATCH_MAX_QUEUE_AGE_MINUTES, + WATCH_MAX_QUEUE_THRESHOLD, + WATCH_STALL_TICKS_DEFAULT, + watchSpecSchema, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; +import { + clampCadence, + variantsOf, + watchDraftError, + watchDraftFor, + withAgeMinutes, + withCadence, + withFollowUp, + withThreshold, + withVariant, + withWindow, +} from "./watch-card"; +import { + watchConditionLabel, + watchConfirmationBlockBody, + watchDurationLabel, + watchOneShotBlockBody, + watchSubjectLabel, +} from "~/presenters/v3/dashboardAgent"; +import { + errorWatchRecommendation, + healthWatchRecommendation, + queueWatchRecommendation, + runWatchRecommendation, +} from "./watch-recommendations"; + +const queueDraft = () => watchDraftFor(queueWatchRecommendation("email-sends")); +const runDraft = () => watchDraftFor(runWatchRecommendation("run_abc123")); + +describe("the recommendations", () => { + it("gives every entry point a spec the schema accepts", () => { + const specs: WatchSpec[] = [ + runWatchRecommendation("run_abc123"), + queueWatchRecommendation("email-sends"), + errorWatchRecommendation("error_a1b2c3d4"), + healthWatchRecommendation("crit"), + ]; + for (const spec of specs) { + expect(watchSpecSchema.safeParse(spec).success).toBe(true); + } + }); + + it("recommends the condition §2.1 assigns to each object", () => { + expect(runWatchRecommendation("run_abc123").kind).toBe("run_finished"); + expect(queueWatchRecommendation("email-sends").kind).toBe("queue_oldest_age"); + expect(errorWatchRecommendation("error_a1b2c3d4").kind).toBe("error_recurrence"); + expect(healthWatchRecommendation("warn").kind).toBe("health_recovery"); + }); + + it("switches the queue recommendation to the drain once runs are already late", () => { + const late = queueWatchRecommendation("email-sends", { + oldestWaitMs: OLDEST_WAIT_WARNING_MS, + }); + expect(late).toMatchObject({ + kind: "backlog_drain", + queue: "email-sends", + }); + expect(watchSpecSchema.safeParse(late).success).toBe(true); + }); + + it("stays on the age SLA when the queue is merely busy, or the signal is missing", () => { + expect( + queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS - 1 }).kind + ).toBe("queue_oldest_age"); + expect(queueWatchRecommendation("email-sends", { oldestWaitMs: null }).kind).toBe( + "queue_oldest_age" + ); + expect(queueWatchRecommendation("email-sends", {}).kind).toBe("queue_oldest_age"); + expect(queueWatchRecommendation("email-sends").kind).toBe("queue_oldest_age"); + }); + + it("starts both follow-ups off — consent is never assumed", () => { + expect(runDraft().followUp).toEqual({ + investigateOnAttention: false, + notifyExternally: false, + }); + }); +}); + +describe("cadence limits", () => { + it("lets a run watch poll every minute", () => { + expect(clampCadence("run_finished", 1)).toBe(1); + }); + + it("floors an aggregate watch at five minutes — never a hot loop", () => { + expect(clampCadence("backlog_drain", 1)).toBe(5); + expect(clampCadence("queue_depth_above", 1)).toBe(5); + expect(clampCadence("health_recovery", 1)).toBe(5); + }); + + it("keeps an offered cadence and rounds an unknown one up", () => { + expect(clampCadence("backlog_drain", 15)).toBe(15); + expect(clampCadence("backlog_drain", 7)).toBe(15); + expect(clampCadence("backlog_drain", 999)).toBe(60); + }); + + it("re-clamps when the kind changes under the user", () => { + const swapped = withVariant(withCadence(runDraft(), 1), "backlog_drain"); + expect(swapped.spec.checkEveryMinutes).toBe(5); + expect(watchSpecSchema.safeParse(swapped.spec).success).toBe(true); + }); +}); + +describe("condition variants (§3)", () => { + it("offers the run pair and the whole queue family", () => { + expect(variantsOf(runDraft())).toEqual(["run_finished", "run_failed"]); + expect(variantsOf(queueDraft())).toEqual([ + "backlog_drain", + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", + ]); + expect(variantsOf(watchDraftFor(errorWatchRecommendation("error_a1")))).toHaveLength(1); + expect(variantsOf(watchDraftFor(healthWatchRecommendation("warn")))).toHaveLength(1); + }); + + it("carries the subject and window across a swap, and restates the note", () => { + const draft = withWindow(runDraft(), 6); + const failed = withVariant(draft, "run_failed"); + expect(failed.spec).toMatchObject({ + kind: "run_failed", + runId: "run_abc123", + maxHours: 6, + note: "tell me if run run_abc123 fails", + }); + }); + + it("restates the note when the threshold number changes", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + // Same verb and same SLA format as the card's condition line: both come from + // the presenter's one wording record. + expect(above.spec.note).toBe("tell me if the email-sends queue goes above 500"); + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 90); + expect(age.spec.note).toBe("tell me if runs in email-sends wait longer than 1h 30m"); + }); + + it("gives the threshold variant a usable default", () => { + const above = withVariant(queueDraft(), "queue_depth_above"); + expect(above.spec).toMatchObject({ kind: "queue_depth_above", queue: "email-sends" }); + expect(watchDraftError(above)).toBeNull(); + }); + + it("swaps back without losing the queue", () => { + const roundTrip = withVariant(withVariant(queueDraft(), "queue_depth_above"), "backlog_drain"); + expect(roundTrip.spec).toMatchObject({ kind: "backlog_drain", queue: "email-sends" }); + }); + + it("gives every queue variant a submittable default and keeps the subject", () => { + for (const kind of [ + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", + ] as const) { + const swapped = withVariant(queueDraft(), kind); + expect(swapped.spec).toMatchObject({ kind, queue: "email-sends" }); + expect(watchDraftError(swapped)).toBeNull(); + expect(watchSpecSchema.safeParse(swapped.spec).success).toBe(true); + } + }); + + it("carries a typed threshold between the two threshold questions", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + const below = withVariant(above, "queue_depth_below"); + expect(below.spec).toMatchObject({ kind: "queue_depth_below", threshold: 500 }); + }); + + it("keeps the stall count internal — the default, never a field", () => { + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(stalled.spec).toMatchObject({ ticks: WATCH_STALL_TICKS_DEFAULT }); + expect(withThreshold(stalled, 5)).toEqual(stalled); + expect(withAgeMinutes(stalled, 5)).toEqual(stalled); + }); +}); + +describe("the window", () => { + it("never leaves the 24-hour ceiling", () => { + expect(withWindow(runDraft(), 999).spec.maxHours).toBe(24); + }); + + it("never goes below the shortest offered window", () => { + expect(withWindow(runDraft(), 0).spec.maxHours).toBe(0.5); + }); +}); + +describe("the follow-up opt-ins (§2.2, binding)", () => { + it("sets them INDEPENDENTLY — never as a radio group", () => { + const both = withFollowUp(withFollowUp(runDraft(), { notifyExternally: true }), { + investigateOnAttention: true, + }); + expect(both.followUp).toEqual({ investigateOnAttention: true, notifyExternally: true }); + }); + + it("turning one off leaves the other alone", () => { + const draft = withFollowUp(runDraft(), { + investigateOnAttention: true, + notifyExternally: true, + }); + expect(withFollowUp(draft, { notifyExternally: false }).followUp).toEqual({ + investigateOnAttention: true, + notifyExternally: false, + }); + }); + + it("has no way to express in-chat delivery at all — it is not a choice", () => { + expect(Object.keys(runDraft().followUp).sort()).toEqual([ + "investigateOnAttention", + "notifyExternally", + ]); + }); +}); + +describe("validation stays inside the card", () => { + it("accepts every recommendation as it opens", () => { + expect(watchDraftError(runDraft())).toBeNull(); + expect(watchDraftError(queueDraft())).toBeNull(); + }); + + it("refuses a half-typed threshold", () => { + const draft = withThreshold(withVariant(queueDraft(), "queue_depth_above"), Number.NaN); + expect(watchDraftError(draft)).toMatch(/whole number/i); + }); + + it("refuses a threshold above the queue-watch ceiling", () => { + const draft = withThreshold( + withVariant(queueDraft(), "queue_depth_above"), + WATCH_MAX_QUEUE_THRESHOLD + 1 + ); + expect(watchDraftError(draft)).toMatch(/too high/i); + }); + + it("ignores a threshold set on a kind that has none", () => { + expect(withThreshold(runDraft(), 5)).toEqual(runDraft()); + }); + + it("refuses a half-typed threshold on the `below` variant too", () => { + const draft = withThreshold(withVariant(queueDraft(), "queue_depth_below"), Number.NaN); + expect(watchDraftError(draft)).toMatch(/whole number/i); + }); + + it("refuses an SLA that is empty, zero, or longer than a watch can run", () => { + const age = withVariant(queueDraft(), "queue_oldest_age"); + expect(watchDraftError(withAgeMinutes(age, Number.NaN))).toMatch(/whole number of minutes/i); + expect(watchDraftError(withAgeMinutes(age, 0))).toMatch(/whole number of minutes/i); + expect(watchDraftError(withAgeMinutes(age, WATCH_MAX_QUEUE_AGE_MINUTES + 1))).toMatch( + /longer than a watch can run/i + ); + expect(watchDraftError(withAgeMinutes(age, 30))).toBeNull(); + }); + + it("ignores an SLA set on a kind that has none", () => { + expect(withAgeMinutes(runDraft(), 5)).toEqual(runDraft()); + }); +}); + +describe("the card's copy", () => { + it("names the subject the way the object does", () => { + expect(watchSubjectLabel(queueWatchRecommendation("email-sends"))).toBe("email-sends"); + expect(watchSubjectLabel(runWatchRecommendation("run_abc123"))).toBe("run run_abc123"); + expect(watchSubjectLabel(healthWatchRecommendation("warn"))).toBe("health"); + }); + + it("states the condition and the duration as §2.2 writes them", () => { + const spec = queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS }); + expect(watchConditionLabel(spec)).toBe("Until the queue drains"); + expect(watchDurationLabel(spec)).toBe("For 1 hour · checking every 5 min"); + }); + + it("carries the threshold into the condition line", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + expect(watchConditionLabel(above.spec)).toBe("If the queue goes above 500"); + }); + + it("states each new queue condition the way the user reads it", () => { + const below = withThreshold(withVariant(queueDraft(), "queue_depth_below"), 100); + expect(watchConditionLabel(below.spec)).toBe("Until the queue is back below 100"); + + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(watchConditionLabel(stalled.spec)).toBe("If the queue stops moving"); + + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 90); + expect(watchConditionLabel(age.spec)).toBe("If runs wait longer than 1h 30m"); + expect(watchSubjectLabel(age.spec)).toBe("email-sends"); + }); + + it("writes the confirmation as one sentence for every queue condition", () => { + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(watchConfirmationBlockBody({ spec: stalled.spec, watchId: "w" }).headline).toBe( + "Watching email-sends in case it stops moving." + ); + + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 5); + expect(watchConfirmationBlockBody({ spec: age.spec, watchId: "w" }).headline).toBe( + "Watching email-sends in case runs wait longer than 5m." + ); + + const below = withThreshold(withVariant(queueDraft(), "queue_depth_below"), 100); + expect(watchConfirmationBlockBody({ spec: below.spec, watchId: "w" }).headline).toBe( + "Watching email-sends until it is back below 100." + ); + }); +}); + +describe("the persisted blocks (§2.2)", () => { + it("states all four lifetime facts on a confirmation", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS }), + watchId: "watch_1", + }); + expect(body.outcome).toBe("watching"); + expect(body.headline).toBe("Watching email-sends until the queue drains."); + expect(body.lifetime).toBe( + "Checking every 5 min for up to 1 hour. It reports once, then stops." + ); + expect(body.watchId).toBe("watch_1"); + expect(body.detail).toBeNull(); + }); + + it("says plainly when the creation-time check couldn't run", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + unavailable: true, + }); + expect(body.detail).toBe("We couldn't check that just now. Watching anyway."); + }); + + it("only claims a follow-up that actually took effect", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + followUp: { investigateOnAttention: true, external: { status: "not_requested" } }, + }); + expect(body.followUp).toEqual(["If it turns out badly, I'll investigate straight away."]); + }); + + it("says out loud when the email the user asked for couldn't be added", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + followUp: { external: { status: "unavailable", reason: "email_alerts_not_configured" } }, + }); + expect(body.followUp).toEqual([ + "I couldn't add email notifications, so updates will appear in the dashboard only.", + ]); + }); + + it("makes a one-shot result carry no lifetime and no watch", () => { + const satisfied = watchOneShotBlockBody({ + spec: queueWatchRecommendation("email-sends"), + result: "satisfied", + }); + expect(satisfied.outcome).toBe("already_true"); + expect(satisfied.headline).toBe("That already happened, so there's nothing left to watch."); + expect(satisfied.lifetime).toBeNull(); + expect(satisfied.watchId).toBeNull(); + + const impossible = watchOneShotBlockBody({ + spec: runWatchRecommendation("run_abc123"), + result: "terminal_unsatisfied", + }); + expect(impossible.outcome).toBe("impossible"); + expect(impossible.headline).toBe("That can't happen any more, so there's nothing to watch."); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-card.ts b/apps/webapp/app/components/dashboard-agent/watch-card.ts new file mode 100644 index 00000000000..9d2a9d54425 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card.ts @@ -0,0 +1,181 @@ +/** + * The watch card's state machine, kept pure so its rules are testable without a DOM. + * + * The card never invents a value the schema would reject: switching condition + * variant re-clamps the cadence, and the window is always one of the offered + * options. The option lists are read from contracts (`watchCadenceOptions`, + * `WATCH_WINDOW_HOURS_OPTIONS`) rather than re-typed, so a picker cannot offer + * something validation would refuse. Nothing here persists: a draft is client-side + * until `Start watching` submits it. + */ +import { + WATCH_DEFAULT_QUEUE_AGE_MINUTES, + WATCH_DEFAULT_QUEUE_THRESHOLD, + WATCH_MAX_HOURS, + WATCH_MAX_QUEUE_AGE_MINUTES, + WATCH_MAX_QUEUE_THRESHOLD, + WATCH_STALL_TICKS_DEFAULT, + WATCH_WINDOW_HOURS_OPTIONS, + watchCadenceOptions, + watchConditionVariants, + watchSpecSchema, + type WatchDraft, + type WatchFollowUp, + type WatchKind, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { noteFor } from "~/presenters/v3/dashboardAgent"; + +/** A brand-new draft: the recommendation, with both opt-ins off. */ +export function watchDraftFor(spec: WatchSpec): WatchDraft { + return { spec, followUp: { investigateOnAttention: false, notifyExternally: false } }; +} + +/** + * The nearest cadence this kind is allowed to poll at. A 1-minute run watch + * switched to a queue variant must land on 5, not fail validation on submit. + */ +export function clampCadence(kind: WatchKind, minutes: number): number { + const options = watchCadenceOptions(kind); + if (options.includes(minutes)) return minutes; + return options.find((option) => option >= minutes) ?? options[options.length - 1]!; +} + +/** + * Swap the condition for its sibling variant, carrying everything else except the + * note, which is restated to describe the new condition. + */ +export function withVariant(draft: WatchDraft, kind: WatchKind): WatchDraft { + const next = variantSpec(draft, kind); + if (next === draft.spec) return draft; + return { ...draft, spec: { ...next, note: noteFor(next) } as WatchSpec }; +} + +function variantSpec(draft: WatchDraft, kind: WatchKind): WatchSpec { + const { spec } = draft; + const common = { + note: spec.note, + maxHours: spec.maxHours, + checkEveryMinutes: clampCadence(kind, spec.checkEveryMinutes), + } as const; + + switch (kind) { + case "run_finished": + case "run_failed": + case "run_start": { + const runId = "runId" in spec ? spec.runId : ""; + return { ...common, kind, runId } as WatchSpec; + } + case "backlog_drain": { + const queue = "queue" in spec ? spec.queue : ""; + return { ...common, kind, queue } as WatchSpec; + } + case "queue_depth_above": + case "queue_depth_below": { + const queue = "queue" in spec ? spec.queue : ""; + // The number carries across the two threshold questions: someone who typed + // 500 for "above" means the same 500 when they flip to "back below". + const threshold = "threshold" in spec ? spec.threshold : WATCH_DEFAULT_QUEUE_THRESHOLD; + return { ...common, kind, queue, threshold } as WatchSpec; + } + case "queue_stalled": { + const queue = "queue" in spec ? spec.queue : ""; + // Ticks are not user-facing: the card never shows a field for them. + const ticks = "ticks" in spec ? spec.ticks : WATCH_STALL_TICKS_DEFAULT; + return { ...common, kind, queue, ticks } as WatchSpec; + } + case "queue_oldest_age": { + const queue = "queue" in spec ? spec.queue : ""; + const thresholdMinutes = + "thresholdMinutes" in spec ? spec.thresholdMinutes : WATCH_DEFAULT_QUEUE_AGE_MINUTES; + return { ...common, kind, queue, thresholdMinutes } as WatchSpec; + } + // The kinds with no second question keep the draft untouched. + default: + return draft.spec; + } +} + +/** + * The conditions this draft's picker offers, in order, including the current one. + * A single-entry list means the kind has no second question and the card states + * the condition as a fact instead of a choice. + */ +export function variantsOf(draft: WatchDraft): readonly WatchKind[] { + return watchConditionVariants(draft.spec.kind); +} + +export function withCadence(draft: WatchDraft, minutes: number): WatchDraft { + return { + ...draft, + spec: { + ...draft.spec, + checkEveryMinutes: clampCadence(draft.spec.kind, minutes), + } as WatchSpec, + }; +} + +export function withWindow(draft: WatchDraft, maxHours: number): WatchDraft { + const clamped = Math.min(Math.max(maxHours, WATCH_WINDOW_HOURS_OPTIONS[0]), WATCH_MAX_HOURS); + return { ...draft, spec: { ...draft.spec, maxHours: clamped } as WatchSpec }; +} + +/** + * The threshold, as the user is typing it. No range checks here: a half-typed + * field is a draft, and `watchDraftError` is what refuses to submit it. + */ +export function withThreshold(draft: WatchDraft, threshold: number): WatchDraft { + if (draft.spec.kind !== "queue_depth_above" && draft.spec.kind !== "queue_depth_below") { + return draft; + } + // The note quotes the number, so a new number restates the note. + const spec = { ...draft.spec, threshold }; + return { ...draft, spec: { ...spec, note: noteFor(spec) } }; +} + +/** The age SLA in minutes, as the user is typing it. Same rule as the threshold. */ +export function withAgeMinutes(draft: WatchDraft, thresholdMinutes: number): WatchDraft { + if (draft.spec.kind !== "queue_oldest_age") return draft; + // The note quotes the number, so a new number restates the note. + const spec = { ...draft.spec, thresholdMinutes }; + return { ...draft, spec: { ...spec, note: noteFor(spec) } }; +} + +/** + * The two follow-up opt-ins, set independently. There is no way to express + * "external instead of chat": in-chat delivery is not a choice, so it is not here. + */ +export function withFollowUp(draft: WatchDraft, patch: Partial): WatchDraft { + return { ...draft, followUp: { ...draft.followUp, ...patch } }; +} + +/** + * Why this draft can't be submitted, in the user's words, or null when it can. + * The schema is the authority, so the card and the server agree by construction; + * this only translates its refusal into the sentence the card shows inline. + */ +export function watchDraftError(draft: WatchDraft): string | null { + if (draft.spec.kind === "queue_depth_above" || draft.spec.kind === "queue_depth_below") { + const { threshold } = draft.spec; + if (!Number.isInteger(threshold) || threshold < 0) { + return "Enter a whole number to watch for."; + } + if (threshold > WATCH_MAX_QUEUE_THRESHOLD) { + return `That threshold is too high — ${WATCH_MAX_QUEUE_THRESHOLD.toLocaleString()} is the most a queue watch takes.`; + } + } + + if (draft.spec.kind === "queue_oldest_age") { + const { thresholdMinutes } = draft.spec; + if (!Number.isInteger(thresholdMinutes) || thresholdMinutes < 1) { + return "Enter a whole number of minutes to watch for."; + } + if (thresholdMinutes > WATCH_MAX_QUEUE_AGE_MINUTES) { + return `That's longer than a watch can run — ${WATCH_MAX_QUEUE_AGE_MINUTES} minutes is the most.`; + } + } + + return watchSpecSchema.safeParse(draft.spec).success + ? null + : "Something in this watch isn't valid. Check the duration and the condition."; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts b/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts new file mode 100644 index 00000000000..4734d5966db --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts @@ -0,0 +1,107 @@ +import { watchIdentity, type WatchSpec } from "@internal/dashboard-agent-contracts"; +import { describe, expect, it } from "vitest"; +import { immediateWatchMessage, watchChipLabel, watchChipTooltip } from "./watch-chips"; + +const chip = (spec: WatchSpec) => ({ + kind: spec.kind, + identity: watchIdentity(spec), + note: spec.note, +}); + +describe("watchChipLabel", () => { + it("labels a run watch with its run id", () => { + expect( + watchChipLabel( + chip({ + kind: "run_finished", + runId: "run_abc123", + note: "Tell me when the retry finishes.", + maxHours: 2, + checkEveryMinutes: 1, + }) + ) + ).toBe("run_abc123"); + }); + + it("labels a backlog watch with the queue name", () => { + expect( + watchChipLabel( + chip({ + kind: "backlog_drain", + queue: "task/send-email", + note: "Tell me when the backlog clears.", + maxHours: 6, + checkEveryMinutes: 5, + }) + ) + ).toBe("task/send-email"); + }); + + it("shortens an error fingerprint", () => { + expect( + watchChipLabel( + chip({ + kind: "error_recurrence", + fingerprint: "0123456789abcdef0123456789abcdef", + note: "Tell me if the rate-limit error comes back.", + maxHours: 12, + checkEveryMinutes: 15, + }) + ) + ).toBe("01234567"); + }); + + it("labels a health watch by its kind, not its report", () => { + expect( + watchChipLabel( + chip({ + kind: "health_recovery", + report: "health", + fromSeverity: "crit", + note: "prod health back to normal", + maxHours: 4, + checkEveryMinutes: 15, + }) + ) + ).toBe("health"); + }); + + it("falls back to the first words of the note when the identity is unreadable", () => { + expect( + watchChipLabel({ kind: "run_start", identity: "nonsense", note: "Tell me when it starts" }) + ).toBe("Tell me when"); + }); + + it("falls back to the kind when there is no note either", () => { + expect(watchChipLabel({ kind: "run_start", identity: "", note: " " })).toBe("run_start"); + }); +}); + +describe("watchChipTooltip", () => { + it("carries the note, the cadence and the state", () => { + expect( + watchChipTooltip({ + note: "Tell me when prod recovers.", + checkEveryMinutes: 15, + status: "active", + }) + ).toBe("Tell me when prod recovers. · every 15 min · watching"); + }); + + it("drops an empty note rather than leaving a dangling separator", () => { + expect(watchChipTooltip({ note: "", checkEveryMinutes: 5, status: "fired" })).toBe( + "every 5 min · fired" + ); + }); +}); + +describe("immediateWatchMessage", () => { + it("says the condition already resolved", () => { + expect(immediateWatchMessage("satisfied")).toMatch(/already happened/); + expect(immediateWatchMessage("terminal_unsatisfied")).toMatch(/can't happen any more/); + }); + + it("never falls through to nothing", () => { + expect(immediateWatchMessage("something-new")).toBeTruthy(); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-chips.ts b/apps/webapp/app/components/dashboard-agent/watch-chips.ts new file mode 100644 index 00000000000..10d9422fc35 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-chips.ts @@ -0,0 +1,74 @@ +/** + * The pure text of the watch UI: a chip's label and its tooltip. + * + * A chip has one line of room in a 380px panel, so the label names the thing being + * watched and the icon carries the state. The label comes from the watch `identity`, + * the same dedup key the store uses, so a chip cannot disagree with the store about + * what it watches. + */ +import type { WatchStatus } from "@internal/dashboard-agent-contracts"; + +// The immediate-check wording lives in the presenter with the rest of the +// user-facing copy. Re-exported here for chip callers. +export { immediateWatchMessage } from "~/presenters/v3/dashboardAgent"; + +import { formatWatchCadence, watchIdentityValue } from "~/presenters/v3/dashboardAgent"; + +export const WATCH_STATUS_LABEL: Record = { + active: "watching", + fired: "fired", + expired: "expired", + cancelled: "cancelled", +}; + +/** Fingerprints are hashes — a chip shows just enough of one to tell them apart. */ +const FINGERPRINT_CHARS = 8; + +/** + * The chip label for a watch. `identity` is `{kind}:{value}`, so the value is the + * thing being watched; a health watch has no per-instance value, so its kind is + * the label. Falls back to the note (then the kind) if the identity is unreadable. + */ +export function watchChipLabel(watch: { kind: string; identity: string; note: string }): string { + const value = watch.identity.startsWith(`${watch.kind}:`) + ? watch.identity.slice(watch.kind.length + 1) + : ""; + + switch (watch.kind) { + case "run_start": + case "run_finished": + case "run_failed": + case "backlog_drain": + case "queue_stalled": + return value || fallbackLabel(watch); + // Identity is `{kind}:{queue}:{number}` here. The chip names the queue; the + // number goes in the tooltip's note, where there is room for it. + case "queue_depth_above": + case "queue_depth_below": + case "queue_oldest_age": + return watchIdentityValue(watch.kind, watch.identity) || fallbackLabel(watch); + case "error_recurrence": + return value ? value.slice(0, FINGERPRINT_CHARS) : fallbackLabel(watch); + case "health_recovery": + return "health"; + default: + return value || fallbackLabel(watch); + } +} + +/** Last resort: the first few words of the note, else the kind as written. */ +function fallbackLabel(watch: { kind: string; note: string }): string { + const words = watch.note.trim().split(/\s+/).filter(Boolean).slice(0, 3).join(" "); + return words || watch.kind; +} + +/** Everything that didn't fit on the chip: why it exists, and its cadence. */ +export function watchChipTooltip(watch: { + note: string; + checkEveryMinutes: number; + status: WatchStatus; +}): string { + const note = watch.note.trim(); + const cadence = formatWatchCadence(watch.checkEveryMinutes); + return [note, cadence, WATCH_STATUS_LABEL[watch.status]].filter(Boolean).join(" · "); +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts b/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts new file mode 100644 index 00000000000..4ab53c2dc90 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts @@ -0,0 +1,78 @@ +import { + WATCH_DEFAULT_QUEUE_AGE_MINUTES, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; +import { noteFor } from "~/presenters/v3/dashboardAgent"; + +/** Distributes over the spec union, so the kind stays discriminated. */ +type WithoutNote = T extends unknown ? Omit : never; + +/** The note comes from the presenter, so a recommendation reads like an edited one. */ +function withNote(spec: WithoutNote): WatchSpec { + const draft = { ...spec, note: "" } as WatchSpec; + return { ...draft, note: noteFor(draft) }; +} + +export function runWatchRecommendation(runFriendlyId: string): WatchSpec { + return withNote({ + kind: "run_finished", + runId: runFriendlyId, + checkEveryMinutes: 1, + maxHours: 1, + }); +} + +/** + * The recommendation must be a condition that isn't true yet: an already-true watch + * one-shots instead of watching. Past the wait threshold that means the drain, not the SLA. + */ +export function queueWatchRecommendation( + queueName: string, + context?: { oldestWaitMs?: number | null } +): WatchSpec { + const oldestWaitMs = context?.oldestWaitMs ?? null; + if (oldestWaitMs !== null && oldestWaitMs >= OLDEST_WAIT_WARNING_MS) { + return withNote({ + kind: "backlog_drain", + queue: queueName, + checkEveryMinutes: 5, + maxHours: 1, + }); + } + + return queueAgeWatchRecommendation(queueName); +} + +export function queueAgeWatchRecommendation( + queueName: string, + thresholdMinutes: number = WATCH_DEFAULT_QUEUE_AGE_MINUTES +): WatchSpec { + return withNote({ + kind: "queue_oldest_age", + queue: queueName, + thresholdMinutes, + checkEveryMinutes: 5, + maxHours: 1, + }); +} + +export function errorWatchRecommendation(errorFriendlyId: string): WatchSpec { + return withNote({ + kind: "error_recurrence", + fingerprint: errorFriendlyId, + checkEveryMinutes: 5, + maxHours: 6, + }); +} + +/** Only offered on a degraded report. `fromSeverity` is what the recovery is measured from. */ +export function healthWatchRecommendation(fromSeverity: "warn" | "crit"): WatchSpec { + return withNote({ + kind: "health_recovery", + report: "health", + fromSeverity, + checkEveryMinutes: 5, + maxHours: 2, + }); +} diff --git a/apps/webapp/app/components/queues/queue-thresholds.ts b/apps/webapp/app/components/queues/queue-thresholds.ts index 43cf2be32e7..1c714bc4035 100644 --- a/apps/webapp/app/components/queues/queue-thresholds.ts +++ b/apps/webapp/app/components/queues/queue-thresholds.ts @@ -1,2 +1,2 @@ -/** Head-of-line wait at which a queue reads as stuck. */ +/** Head-of-line wait at which a queue reads as stuck. Shared by the queue page and the watch card. */ export const OLDEST_WAIT_WARNING_MS = 5 * 60_000; diff --git a/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts index 83ab09c177c..dff916a6aa1 100644 --- a/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts @@ -18,6 +18,7 @@ export const ApiAlertType = z.enum([ "deployment_failure", "deployment_success", "error_group", + "dashboard_agent_watch", ]); export type ApiAlertType = z.infer; @@ -88,6 +89,8 @@ export class ApiAlertChannelPresenter { return "deployment_success"; case "ERROR_GROUP": return "error_group"; + case "DASHBOARD_AGENT_WATCH": + return "dashboard_agent_watch"; default: assertNever(alertType); } @@ -105,6 +108,8 @@ export class ApiAlertChannelPresenter { return "DEPLOYMENT_SUCCESS"; case "error_group": return "ERROR_GROUP"; + case "dashboard_agent_watch": + return "DASHBOARD_AGENT_WATCH"; default: assertNever(alertType); } diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts b/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts new file mode 100644 index 00000000000..df3c791a82e --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts @@ -0,0 +1,98 @@ +/** + * A view block or a resolved watch as plain text. + * + * The panel renders blocks as React; an email, a Slack message, a webhook body or + * a log line cannot. Rather than each of those re-saying the block's contents in + * its own words, they render it here. Pure, no React, no request context. + */ +import type { ViewBlock } from "@internal/dashboard-agent-contracts"; +import { presentResolvedWatch, watchNoteLine, type WatchResolvedInput } from "./watch-wording"; + +/** A labelled scalar the check observed. */ +export type TextFact = { label: string; value: string }; + +/** Facts as one `Label: value` line each. */ +export function renderFactLines(facts: readonly TextFact[]): string[] { + return facts.map((fact) => `${fact.label}: ${fact.value}`); +} + +function lines(...parts: Array): string { + return parts + .filter((part): part is string => typeof part === "string" && part.length > 0) + .join("\n"); +} + +/** + * One view block as plain text. Says what the card says and nothing more — no + * surface may add a sentence of its own on top. + */ +export function renderBlockAsText(block: ViewBlock): string { + switch (block.type) { + case "watch_result": + return lines(block.headline, block.lifetime, block.detail, ...block.followUp); + + case "diagnosis": + return lines( + block.summary, + `Likely cause: ${block.likelyCause}`, + `Confidence: ${block.confidence}`, + block.impact ? `Impact: ${block.impact}` : null, + ...block.evidence.map( + (item) => + `Evidence (${item.type}): ${item.detail}${item.reference ? ` — ${item.reference}` : ""}` + ), + ...block.nextSteps.map((step, index) => `${index + 1}. ${step}`) + ); + + case "investigation": { + const state = block.investigation; + return lines( + state.title, + state.headline, + `Outcome: ${state.outcome} · severity ${state.severity} · confidence ${state.confidence}`, + ...state.hypotheses.map( + (hypothesis) => + `${hypothesis.statement} — ${hypothesis.verdict}${ + hypothesis.finding ? `: ${hypothesis.finding}` : "" + }` + ), + state.remediation ? `Fix: ${state.remediation}` : null, + ...(state.checkNext ?? []).map((step) => `Check next: ${step}`), + state.caveat ? `Caveat: ${state.caveat.message}` : null + ); + } + + case "report": { + const { vm } = block; + return lines( + `${vm.title} report for ${vm.scope} (${vm.period}): ${vm.summary.severity}`, + ...vm.findings.map((finding) => `${finding.type} — ${finding.severity}: ${finding.reason}`) + ); + } + + // A chart is its shape, not its rows: the rows come from running the query. + case "chart": + return lines(`Chart: ${block.title ?? "untitled"} (${block.chartType})`, block.query); + + case "actions": + return lines(...block.actions.map((action) => `- ${action.label}`)); + + default: { + const unreachable: never = block; + throw new Error(`Unhandled view block: ${JSON.stringify(unreachable)}`); + } + } +} + +/** + * A resolved watch as plain text: the fact, why it was being watched, then what the + * resolving check saw. What the email body and the Slack message both say. + */ +export function renderResolvedWatchAsText(args: { + resolved: WatchResolvedInput; + note: string; + facts: readonly TextFact[]; +}): string { + const { headline } = presentResolvedWatch(args.resolved); + return lines(headline, watchNoteLine(args.note), ...renderFactLines(args.facts)); +} diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/index.ts b/apps/webapp/app/presenters/v3/dashboardAgent/index.ts new file mode 100644 index 00000000000..07d781dc6db --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/index.ts @@ -0,0 +1,5 @@ +// The dashboard agent's presenter: the one place a watch, a view block or a watch +// result becomes English. Every surface (card, banner, toast, email, Slack, +// webhook) imports from here. +export * from "./block-text"; +export * from "./watch-wording"; diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts b/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts new file mode 100644 index 00000000000..f97e517301b --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts @@ -0,0 +1,38 @@ +/** + * The watch vocabulary moved into the contracts package so the agent's own + * deterministic narration says the same sentences the dashboard does — the agent + * cannot import the webapp, and a second vocabulary would drift within a release. + * + * Re-exported here because every webapp surface imports the presenter, not contracts. + */ +export { + formatWatchCadence, + formatWatchDuration, + formatWatchSla, + formatWatchWait, + formatWatchWindow, + immediateWatchMessage, + noteFor, + presentResolvedWatch, + WATCH_IN_CHAT_DELIVERY_LINE, + WATCH_PRESENTATION_FALLBACK, + WATCH_UPDATE_LABEL, + watchConditionLabel, + watchConditionWording, + watchConfirmationBlockBody, + watchDurationLabel, + watchExternalNotificationLine, + watchFollowUpLines, + watchIdentityValue, + watchLifetimeSentence, + watchNoteLine, + watchOneShotBlockBody, + watchRequestSentence, + watchSubjectLabel, + watchSubline, + watchTooltipLabel, + type WatchConditionWording, + type WatchPresentation, + type WatchResolvedInput, + type WatchSemanticIcon, +} from "@internal/dashboard-agent-contracts"; diff --git a/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts b/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts index d6a8f2ebd9c..fbe416db338 100644 --- a/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts @@ -11,7 +11,7 @@ import { type ReportViewModel } from "./report-view-model"; const DEFAULT_PERIOD = "1h"; -/** How long a finished report stays reusable. */ +/** How long a finished report stays reusable. Must stay under the watch tick cadence. */ export const REPORT_CACHE_TTL_MS = 90_000; /** How many report, environment and period triples one instance keeps. */ diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx index e0ee26255b7..41dd31c5d17 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx @@ -53,9 +53,13 @@ export const meta = pageMeta("New alert"); const FormSchema = z .object({ alertTypes: z - .array(z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"])) + .array( + z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS", "DASHBOARD_AGENT_WATCH"]) + ) .min(1) - .or(z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"])), + .or( + z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS", "DASHBOARD_AGENT_WATCH"]) + ), environmentTypes: z .array(z.enum(["STAGING", "PRODUCTION", "PREVIEW"])) .min(1) @@ -456,6 +460,18 @@ export default function Page() { defaultChecked /> +
+ + +
+ {alertTypes.errors} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx index a2d77e99edc..7ce5500a875 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx @@ -570,6 +570,8 @@ export function alertTypeTitle(alertType: ProjectAlertType): string { return "Deployment success"; case "ERROR_GROUP": return "Error group"; + case "DASHBOARD_AGENT_WATCH": + return "Dashboard agent watches"; default: { throw new Error(`Unknown alertType: ${alertType}`); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index ff69c89c52b..1946312aac3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -24,6 +24,8 @@ import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; import { RunsIcon } from "~/assets/icons/RunsIcon"; import { CodeBlock } from "~/components/code/CodeBlock"; import { InvestigateButton } from "~/components/dashboard-agent/InvestigateButton"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { errorWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; import { errorGroupPrompt } from "~/components/dashboard-agent/investigate-prompts"; import { ErrorStatusBadge } from "~/components/errors/ErrorStatusBadge"; import { @@ -586,7 +588,7 @@ function ErrorDetailSidebar({
Details - {/* Self-hides when the agent isn't available. */} + {/* Both buttons self-hide when the agent isn't available. */}
+
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 38434f1cdf6..d10eea6cc40 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -8,6 +8,8 @@ import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { AnimatedOrgBannerBar } from "~/components/billing/AnimatedOrgBannerBar"; import { BigNumber } from "~/components/metrics/BigNumber"; import { Header3 } from "~/components/primitives/Headers"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { queueWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; import { OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { Spinner } from "~/components/primitives/Spinner"; @@ -331,7 +333,8 @@ export default function Page() { maxPeriodDays={maxPeriodDays} shortcut={{ key: "d" }} /> - {/* Self-hides when the agent isn't available. */} + {/* Both buttons self-hide when the agent isn't available. Watch is + pre-filled with this queue's recommendation. */} {degraded ? ( ) : null} + { }) : null; + // One narrow read per page load, so the wake signal reaches a browser that has never opened + // the panel — including one whose watch hasn't fired yet. The poll never asks for this. + let dashboardAgentActivity: DashboardAgentWakeActivity = { + unreadWakes: 0, + hasActiveWatches: false, + }; + if (hasDashboardAgentAccess) { + try { + dashboardAgentActivity = await readDashboardAgentWakeActivity(dashboardAgentDb, { + organizationId: project.organization.id, + userId: user.id, + }); + } catch (error) { + // The dashboard must load even when the agent's store doesn't answer. + logger.error("Failed to read dashboard agent wake activity", { error }); + } + } + return { ...project, hasDashboardAgentAccess, promotedDashboardAgentPrompt, + dashboardAgentActivity, }; }; export default function Page() { - const { hasDashboardAgentAccess, promotedDashboardAgentPrompt } = useLoaderData(); + const { hasDashboardAgentAccess, promotedDashboardAgentPrompt, dashboardAgentActivity } = + useLoaderData(); return ( diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts new file mode 100644 index 00000000000..10cf14d6abb --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts @@ -0,0 +1,100 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { resolveAgentAlertContext } from "~/services/dashboardAgentAlertContext.server"; +import { unsubscribeChannelFromWatchAlerts } from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; + +/** + * `DELETE /api/v1/dashboard-agent/alerts/:channelId` — stop alerting this channel + * when a watch fires. The channel is looked up scoped to the chat's project. + */ + +const ParamsSchema = z.object({ channelId: z.string().min(1) }); + +const BodySchema = z.object({ + chatId: z.string().min(1), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "DELETE") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const authentication = await authenticateUatOrApiRequest(request); + if (!authentication?.userActor) { + return json({ error: "Invalid or missing access token" }, { status: 401 }); + } + if (authentication.userActor.client !== "dashboard-agent") { + return json({ error: "Not allowed", code: "forbidden_client" }, { status: 403 }); + } + const userId = authentication.userActor.userId; + // The turn's environment scope is the authority for the chat's project below. + const environmentId = authentication.userActor.environmentId; + if (!environmentId) { + return json( + { error: "This chat has no environment context.", code: "invalid_target" }, + { status: 400 } + ); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + const body = parsedBody.data; + + try { + const context = await resolveAgentAlertContext({ + userId, + environmentId, + chatId: body.chatId, + claimedEnvironmentId: body.environmentId, + claimedProjectRef: body.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: context.code === "environment_mismatch" ? 400 : 404 } + ); + } + + const result = await unsubscribeChannelFromWatchAlerts(parsedParams.data.channelId, { + projectId: context.environment.project.id, + // A project is shared by every member, so the caller's own address is part of the scope. + organizationId: context.environment.organizationId, + ownerUserId: userId, + }); + if (!result.ok) { + if (result.reason === "conflict") { + return json( + { error: "That alert was being changed elsewhere. Try again.", code: "conflict" }, + { status: 409 } + ); + } + return json({ error: "Alert not found", code: "not_found" }, { status: 404 }); + } + + return json({ ok: true, disabledChannel: result.disabledChannel }); + } catch (error) { + logger.error("Failed to unsubscribe a channel from dashboard agent watch alerts", { + error, + userId, + environmentId, + channelId: parsedParams.data.channelId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts new file mode 100644 index 00000000000..cde67806e67 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts @@ -0,0 +1,220 @@ +import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { $replica, prisma } from "~/db.server"; +import { + ProjectAlertEmailProperties, + ProjectAlertSlackProperties, +} from "~/models/projectAlert.server"; +import { + resolveAgentAlertContext, + type AgentAlertContextError, +} from "~/services/dashboardAgentAlertContext.server"; +import { + canUseDashboardAgentEmailAlerts, + DASHBOARD_AGENT_WATCH_ALERT_TYPE, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; +import { CreateAlertChannelService } from "~/v3/services/alerts/createAlertChannel.server"; + +/** + * `GET` lists this chat's project's watch alerts; `POST` subscribes the user's email. Only + * the agent's delegated user-actor token is accepted, and the environment comes from it. + */ + +const ListQuerySchema = z.object({ + chatId: z.string().min(1), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +const CreateBodySchema = z.object({ + chatId: z.string().min(1), + channel: z.literal("email"), + /** May only be the authenticated user's own account email. */ + email: z.string().email().optional(), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +/** A token without an environment scope is unusable here. */ +async function authenticate( + request: Request +): Promise<{ userId: string; environmentId: string } | { error: Response }> { + const authentication = await authenticateUatOrApiRequest(request); + const actor = authentication?.userActor; + if (!actor || actor.client !== "dashboard-agent") { + return { error: json({ error: "Invalid or missing access token" }, { status: 401 }) }; + } + if (!actor.environmentId) { + return { + error: json( + { error: "This chat has no environment context.", code: "invalid_target" }, + { status: 400 } + ), + }; + } + return { userId: actor.userId, environmentId: actor.environmentId }; +} + +/** A mismatched claim is the caller's error, the rest are 404s. */ +function contextStatus(code: AgentAlertContextError) { + return code === "environment_mismatch" ? 400 : 404; +} + +export async function loader({ request }: LoaderFunctionArgs) { + const auth = await authenticate(request); + if ("error" in auth) return auth.error; + + const query = ListQuerySchema.safeParse( + Object.fromEntries(new URL(request.url).searchParams.entries()) + ); + if (!query.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const context = await resolveAgentAlertContext({ + userId: auth.userId, + environmentId: auth.environmentId, + chatId: query.data.chatId, + claimedEnvironmentId: query.data.environmentId, + claimedProjectRef: query.data.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: contextStatus(context.code) } + ); + } + + const channels = await $replica.projectAlertChannel.findMany({ + where: { + projectId: context.environment.project.id, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + }, + select: { id: true, type: true, enabled: true, properties: true, environmentTypes: true }, + orderBy: { createdAt: "asc" }, + }); + + return json({ + alerts: channels.map((channel) => ({ + id: channel.id, + type: channel.type, + enabled: channel.enabled, + environmentTypes: channel.environmentTypes, + target: describeTarget(channel.type, channel.properties), + })), + }); +} + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const auth = await authenticate(request); + if ("error" in auth) return auth.error; + const { userId } = auth; + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const parsed = CreateBodySchema.safeParse(rawBody); + if (!parsed.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + const body = parsed.data; + + const context = await resolveAgentAlertContext({ + userId, + environmentId: auth.environmentId, + chatId: body.chatId, + claimedEnvironmentId: body.environmentId, + claimedProjectRef: body.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: contextStatus(context.code) } + ); + } + const { environment } = context; + + const gate = await canUseDashboardAgentEmailAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + projectId: environment.project.id, + }); + if (!gate.allowed) { + return json({ error: "Alerts are not available here", code: gate.reason }, { status: 403 }); + } + + // Only the signed-in user's own account email may be subscribed. Read off the + // primary: this is the identity the subscription is pinned to. + const user = await prisma.user.findFirst({ where: { id: userId }, select: { email: true } }); + if (!user) { + return json({ error: "User not found", code: "invalid_request" }, { status: 404 }); + } + const email = user.email; + if (body.email && body.email.trim().toLowerCase() !== email.toLowerCase()) { + return json( + { + error: + "Watch alerts can only go to your own account email. Ask the user to add another address on the Alerts page.", + code: "email_not_allowed", + }, + { status: 400 } + ); + } + + try { + const service = new CreateAlertChannelService(); + const channel = await service.call(environment.project.externalRef, userId, { + name: `Watch alerts for ${email}`, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE], + environmentTypes: [environment.type], + // Stable per (email, project), so asking twice re-enables one channel. + deduplicationKey: `dashboard-agent-watch:${email}`, + channel: { type: "EMAIL", email }, + }); + + return json({ id: channel.id, type: channel.type, target: email, enabled: channel.enabled }); + } catch (error) { + // A thrown Response is Remix control flow, not a failure to report. + if (error instanceof Response) throw error; + logger.error("Failed to create a dashboard agent watch alert channel", { + error, + userId, + organizationId: environment.organizationId, + projectId: environment.project.id, + environmentId: environment.id, + }); + return json({ error: "Internal Server Error", code: "internal" }, { status: 500 }); + } +} + +/** A short, non-secret description of where a channel delivers. */ +function describeTarget(type: string, properties: unknown): string | undefined { + if (type === "EMAIL") { + const parsed = ProjectAlertEmailProperties.safeParse(properties); + return parsed.success ? maskEmail(parsed.data.email) : undefined; + } + if (type === "SLACK") { + const parsed = ProjectAlertSlackProperties.safeParse(properties); + return parsed.success ? `#${parsed.data.channelName}` : undefined; + } + // Webhook URLs stay out of the agent's context entirely. + return undefined; +} + +function maskEmail(email: string): string { + const [local, domain] = email.split("@"); + if (!domain || !local) return "an email address"; + const head = local.slice(0, 2); + return `${head}${local.length > 2 ? "…" : ""}@${domain}`; +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts new file mode 100644 index 00000000000..406f0b5b2e5 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts @@ -0,0 +1,191 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { cancelWatch, getWatch, recordWatchCheck } from "@internal/dashboard-agent-db"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { checkWatch, previousCheckFacts } from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { + armDashboardAgentWatchBatch, + authorizeWatchEnvironment, +} from "~/services/dashboardAgentWatches.server"; +import { + WATCH_TOKEN_GRACE_MS, + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * Private per-watch check. The token only names a watch; the row is the authority on + * lifecycle and its snapshot, and this route transitions nothing and advances no tick. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +/** Best-effort: a chain that couldn't be armed returns `false` and is retried next check. */ +async function ensureBatchChain(watch: { + id: string; + environmentId: string; + spec: { checkEveryMinutes: number }; +}): Promise { + try { + const { running } = await armDashboardAgentWatchBatch({ + environmentId: watch.environmentId, + cadenceMinutes: watch.spec.checkEveryMinutes, + }); + return running; + } catch (error) { + logger.error("Dashboard agent watch check: couldn't arm the batch chain", { + watchId: watch.id, + environmentId: watch.environmentId, + error, + }); + return false; + } +} + +const BodySchema = z.object({ + /** The expiry evaluation: allowed after `expiresAt`, within the token's grace. */ + final: z.boolean().optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + // A valid token for a different watch is 403, not 401. + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + let rawBody: unknown; + try { + const raw = await request.text(); + rawBody = raw.length > 0 ? JSON.parse(raw) : {}; + } catch { + return json({ error: "Invalid request body" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) return json({ error: "Invalid request body" }, { status: 400 }); + const body = parsedBody.data; + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + // Terminal watches are never checked again, whatever the token says. + if (watch.status !== "active") { + return json( + { + error: `This watch is ${watch.status}`, + code: watch.status === "cancelled" ? "cancelled" : "not_active", + status: watch.status, + }, + { status: 403 } + ); + } + + const now = new Date(); + const expired = watch.expiresAt.getTime() <= now.getTime(); + if (expired) { + // Past the deadline only the final evaluation is allowed, inside the token's grace. + const graceEnds = watch.expiresAt.getTime() + WATCH_TOKEN_GRACE_MS; + if (body.final !== true || now.getTime() > graceEnds) { + return json( + { error: "This watch has expired", code: "expired", expiresAt: watch.expiresAt }, + { status: 403 } + ); + } + } + + try { + // Re-authorize the initiating user before any environment data is read. + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + // A watch must not outlive the access it was created with. Never notified. + await cancelWatch(dashboardAgentDb, { id: watchId, reason: "access_revoked" }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { status: 403 } + ); + } + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const outcome = await checkWatch( + watch.spec, + watchCheckDeps(authorization.environment, now), + // A tick that couldn't read anything freezes a streak instead of resetting it. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch check failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }) + ); + + // Recorded even on the final evaluation. Guarded on `active`, so a concurrent + // fire/expire wins and this no-ops. + await recordWatchCheck(dashboardAgentDb, { + id: watchId, + lastResult: { + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + final: body.final === true, + }, + }); + + const batched = await ensureBatchChain(watch); + + // `observed` travels with the verdict so no delivery surface re-reads the source. + return json({ + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + batched, + }); + } catch (error) { + logger.error("Dashboard agent watch check tick failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts new file mode 100644 index 00000000000..427435835c1 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts @@ -0,0 +1,113 @@ +import { + claimWatchAlertDispatch, + getWatch, + releaseWatchAlertDispatch, +} from "@internal/dashboard-agent-db"; +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { enqueueWatchFiredAlert } from "~/services/dashboardAgentWatchAlerts.server"; +import { authorizeWatchEnvironment } from "~/services/dashboardAgentWatches.server"; +import { + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; +import { logger } from "~/services/logger.server"; + +/** + * The watcher task reports a fired watch. The row is the authority on whether it fired, and + * the initiating user is re-authorized against its snapshot before any alert is sent. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + // Anything that isn't a fired watch gets no alert, whatever the caller claims. + if (watch.status !== "fired" || !watch.firedAt) { + return json( + { error: `This watch is ${watch.status}`, code: "not_fired", status: watch.status }, + { status: 409 } + ); + } + + try { + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + // Not cancelled here: the watch is already terminal. + logger.info("Dashboard agent watch fired, but access was revoked; no alert", { watchId }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { + status: 403, + } + ); + } + + const claimed = await claimWatchAlertDispatch(dashboardAgentDb, { + id: watch.id, + terminalStatus: "fired", + }); + if (!claimed) { + logger.info("Dashboard agent watch fired callback repeated; no second alert", { watchId }); + return json({ ok: true, alerted: false }); + } + + try { + await enqueueWatchFiredAlert(watch, "fired"); + } catch (error) { + await releaseWatchAlertDispatch(dashboardAgentDb, { id: watch.id, terminalStatus: "fired" }); + throw error; + } + + return json({ ok: true, alerted: true }); + } catch (error) { + logger.error("Dashboard agent watch fire callback failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts new file mode 100644 index 00000000000..83af5eba382 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts @@ -0,0 +1,105 @@ +import { getWatch, isTerminalWatchStatus } from "@internal/dashboard-agent-db"; +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { + kickWatchInvestigation, + watchWantsInvestigation, +} from "~/services/dashboardAgentWatchInvestigate.server"; +import { authorizeWatchEnvironment } from "~/services/dashboardAgentWatches.server"; +import { + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; +import { logger } from "~/services/logger.server"; + +/** + * The watcher task reports a delivered wake for a pre-approved investigation. The caller's + * body is ignored: consent, outcome, user and environment all come off the row. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + if (!isTerminalWatchStatus(watch.status)) { + return json( + { error: `This watch is ${watch.status}`, code: "not_resolved", status: watch.status }, + { status: 409 } + ); + } + + if (!watchWantsInvestigation(watch)) { + // No consent, or an outcome consent doesn't cover: the wake was the whole delivery. + return json({ ok: true, investigating: false }); + } + + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + logger.info("Dashboard agent watch resolved, but access was revoked; no investigation", { + watchId, + }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { status: 403 } + ); + } + + // Never an error to the caller: the wake is already delivered and marked, so a failed + // kick must not make the watcher retry. The stale-investigation sweep settles it. + try { + await kickWatchInvestigation({ watch, environment: authorization.environment }); + } catch (error) { + // A thrown Response is Remix control flow, not a failed kick. + if (error instanceof Response) throw error; + logger.error("Dashboard agent watch investigation could not be started", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + return json({ ok: true, investigating: false, code: "kick_failed" }); + } + + return json({ ok: true, investigating: true }); +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts new file mode 100644 index 00000000000..ac869d68ea5 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts @@ -0,0 +1,75 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { runWatchBatchCheck } from "~/services/dashboardAgentWatchBatch.server"; +import { logger } from "~/services/logger.server"; +import { + bearerToken, + verifyWatchBatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * Private batch check: one call per (environment, cadence) group per cadence. The token + * names the group, the body names the tick. `runWatchBatchCheck` documents the rest. + */ + +const BodySchema = z.object({ + environmentId: z.string().min(1), + cadenceMinutes: z.number().int().positive(), + epoch: z.number().int().nonnegative(), + tick: z.number().int().positive(), +}); + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchBatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + let rawBody: unknown; + try { + const raw = await request.text(); + rawBody = raw.length > 0 ? JSON.parse(raw) : {}; + } catch { + return json({ error: "Invalid request body" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) return json({ error: "Invalid request body" }, { status: 400 }); + const body = parsedBody.data; + + // A valid token for a different group is 403, not 401. + if ( + claims.environmentId !== body.environmentId || + claims.cadenceMinutes !== body.cadenceMinutes + ) { + return json({ error: "Not allowed for this group", code: "group_mismatch" }, { status: 403 }); + } + + try { + return json(await runWatchBatchCheck(body)); + } catch (error) { + logger.error("Dashboard agent watch batch check failed", { + error, + environmentId: body.environmentId, + cadenceMinutes: body.cadenceMinutes, + epoch: body.epoch, + tick: body.tick, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts new file mode 100644 index 00000000000..0fbe83e3463 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts @@ -0,0 +1,163 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { watchSpecSchema } from "@internal/dashboard-agent-contracts"; +import { z } from "zod"; +import { logger } from "~/services/logger.server"; +import { resolveWatchEmailAlertsState } from "~/services/dashboardAgentWatchAlerts.server"; +import { + authorizeWatchEnvironmentById, + createDashboardAgentWatch, + resolveChatWatchContext, +} from "~/services/dashboardAgentWatches.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; + +/** + * Programmatic watch creation (MCP). Only the agent's delegated user-actor token is + * accepted, and the environment comes from it, never the body or the chat's stored context. + */ + +const BodySchema = z.object({ + spec: watchSpecSchema, + chatId: z.string().min(1), + /** Consent for the wake turn to open an investigation. Off unless explicitly sent. */ + investigateOnAttention: z.boolean().optional(), + /** + * Only checked against the token's environment scope, never used in its place. + * `environmentId` is the canonical `RuntimeEnvironment.id`, not a slug. + */ + projectRef: z.string().min(1).optional(), + environmentId: z.string().min(1).optional(), +}); + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const authentication = await authenticateUatOrApiRequest(request); + if (!authentication?.userActor) { + return json({ error: "Invalid or missing access token" }, { status: 401 }); + } + if (authentication.userActor.client !== "dashboard-agent") { + return json({ error: "Not allowed", code: "forbidden_client" }, { status: 403 }); + } + const userId = authentication.userActor.userId; + // The environment this turn is scoped to. There is no trusted fallback. + const environmentId = authentication.userActor.environmentId; + if (!environmentId) { + return json( + { error: "This chat has no environment context to watch in.", code: "invalid_target" }, + { status: 400 } + ); + } + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid watch request", code: "invalid_request" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) { + return json({ error: "Invalid watch request", code: "invalid_request" }, { status: 400 }); + } + const parsed = parsedBody.data; + + // Refuse a body naming a different environment rather than silently picking one. + if (parsed.environmentId && parsed.environmentId !== environmentId) { + return json( + { + error: "That environment isn't the one this chat is open in.", + code: "environment_mismatch", + }, + { status: 400 } + ); + } + + try { + // A chat this user doesn't own does not exist here. + const chat = await resolveChatWatchContext({ chatId: parsed.chatId, userId }); + if (!chat) { + return json({ error: "Chat not found", code: "chat_not_found" }, { status: 404 }); + } + + // The same authorization a background check applies. + const environment = await authorizeWatchEnvironmentById({ userId, environmentId }); + if (!environment) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + // A chat belongs to one org; its watches can't point at another org's env. + if (environment.organizationId !== chat.organizationId) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + // Same check as `environmentId`, for callers that send the project instead. + if (parsed.projectRef && environment.project.externalRef !== parsed.projectRef) { + return json( + { + error: "That project isn't the one this chat is open in.", + code: "environment_mismatch", + }, + { status: 400 } + ); + } + + const result = await createDashboardAgentWatch({ + environment, + userId, + chatId: parsed.chatId, + spec: parsed.spec, + investigateOnAttention: parsed.investigateOnAttention, + }); + + if (!result.ok) { + const status = + result.code === "limit_reached" || result.code === "duplicate" + ? 409 + : result.code === "invalid_target" + ? 404 + : // The chat was deleted while the create was in flight. + result.code === "chat_not_found" + ? 404 + : result.code === "not_configured" + ? 501 + : 500; + return json( + { + error: result.error, + code: result.code, + ...(result.existingId ? { existingId: result.existingId } : {}), + }, + { status } + ); + } + + // One-shot: the immediate check answered, so there is no watch row and no id. + if (!result.watching) { + return json({ + watching: false, + identity: result.identity, + immediate: { result: result.immediate.result, facts: result.immediate.facts }, + }); + } + + return json({ + watching: true, + watchId: result.watchId, + identity: result.identity, + status: result.status, + expiresAt: result.expiresAt.toISOString(), + emailAlerts: await resolveWatchEmailAlertsState({ userId, environment }), + ...(result.unavailable ? { unavailable: true } : {}), + }); + } catch (error) { + // A thrown Response is Remix control flow, not a failure to report. + if (error instanceof Response) throw error; + logger.error("Failed to create a dashboard agent watch", { + error, + userId, + environmentId, + chatId: parsed.chatId, + }); + return json({ error: "Internal Server Error", code: "internal" }, { status: 500 }); + } +} diff --git a/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx b/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx new file mode 100644 index 00000000000..771d326e556 --- /dev/null +++ b/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx @@ -0,0 +1,164 @@ +import { EnvelopeIcon } from "@heroicons/react/24/solid"; +import { Form, useNavigation } from "@remix-run/react"; +import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { typedjson, useTypedActionData, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { AppContainer, MainCenteredContainer } from "~/components/layout/AppLayout"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { FormTitle } from "~/components/primitives/FormTitle"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { prisma } from "~/db.server"; +import { verifyUnsubscribeToken } from "~/services/dashboardAgentAlertUnsubscribeToken.server"; +import { + DASHBOARD_AGENT_WATCH_ALERT_TYPE, + unsubscribeChannelFromWatchAlerts, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { rootPath } from "~/utils/pathBuilder"; + +/** + * The unsubscribe link in a watch alert email. The signed token is the whole authorization + * and names one channel; GET confirms and POST acts, so a link preview can't unsubscribe. + */ + +const ParamsSchema = z.object({ channelId: z.string().min(1) }); + +async function authorize(request: Request, params: Record) { + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return undefined; + + const token = new URL(request.url).searchParams.get("token"); + if (!token) return undefined; + + const claims = await verifyUnsubscribeToken(token); + if (!claims) return undefined; + if (claims.channelId !== parsedParams.data.channelId) return undefined; + if (claims.alertType !== DASHBOARD_AGENT_WATCH_ALERT_TYPE) return undefined; + + return claims; +} + +export async function loader({ request, params }: LoaderFunctionArgs) { + const claims = await authorize(request, params); + // The POST needs the token, and a bare `
` drops search params. + return typedjson({ + valid: claims !== undefined, + formAction: `${new URL(request.url).pathname}${new URL(request.url).search}`, + }); +} + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return typedjson({ success: false as const, message: "Method not allowed" }, { status: 405 }); + } + + const claims = await authorize(request, params); + if (!claims) { + return typedjson( + { + success: false as const, + message: "This link is no longer valid, so we couldn't turn off the alerts.", + }, + { status: 403 } + ); + } + + // Read only for the failure log. The unsubscribe does its own scoped lookup. + const channel = await prisma.projectAlertChannel.findFirst({ + where: { id: claims.channelId }, + select: { projectId: true }, + }); + + try { + const result = await unsubscribeChannelFromWatchAlerts(claims.channelId); + if (!result.ok) { + return result.reason === "conflict" + ? typedjson( + { + success: false as const, + message: "This alert was being changed elsewhere. Please try again.", + }, + { status: 409 } + ) + : typedjson( + { success: false as const, message: "This alert no longer exists." }, + { status: 404 } + ); + } + + return typedjson({ success: true as const, channelName: result.channelName }); + } catch (error) { + logger.error("Failed to turn off watch alerts from an email link", { + error, + channelId: claims.channelId, + projectId: channel?.projectId, + }); + throw error; + } +} + +export default function Page() { + const { valid, formAction } = useTypedLoaderData(); + const result = useTypedActionData(); + const navigation = useNavigation(); + const isLoading = navigation.state !== "idle"; + + if (result?.success) { + return ( + + + {result.channelName} will no longer be alerted when a watch fires. You can turn it back on + from the Alerts page in your project. + + + Dashboard + + + ); + } + + if (!valid || result?.success === false) { + return ( + + + {result?.success === false + ? result.message + : "This link is no longer valid. You can manage alerts from the Alerts page in your project."} + + + Dashboard + + + ); + } + + return ( + + + This stops the alerts this channel receives when a watch you set up with the dashboard agent + fires. Other alerts on the channel are unaffected. + + + + + + ); +} + +function Shell({ title, children }: { title: string; children: React.ReactNode }) { + return ( + + +
+ } + title={title} + /> + {children} +
+
+
+ ); +} diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts index cdeb6f83a04..c7a1b14c0d7 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts @@ -1,16 +1,22 @@ import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { + cancelWatch, chatExists, + countUnreadWatchWakes, countUserMessages, createChat, getChatMessages, getSession, + getWatch, listChatIdsWithOpenInvestigations, + listChatIdsWithUnreadWakes, listChats, + markChatRead, + readWatchWakeFeed, renameChat, setChatPinned, - softDeleteChat, } from "@internal/dashboard-agent-db"; +import { watchDraftSchema, type WatchDraft } from "@internal/dashboard-agent-contracts"; import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import type { UIMessage } from "ai"; import { z } from "zod"; @@ -26,6 +32,12 @@ import { $replica } from "~/db.server"; import { env } from "~/env.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { + authorizeWatchEnvironmentById, + deleteChatWithWatches, + listActiveWatchesForChats, + submitDashboardAgentWatch, +} from "~/services/dashboardAgentWatches.server"; import { dashboardAgentApiOrigin, isDashboardAgentConfigured, @@ -61,8 +73,11 @@ const ActionBody = z.object({ "rename", "pin", "delete", + "read", "resolve", "resolve-many", + "watch-cancel", + "watch-create", ]), // Omitted for `create` (the server generates it); required for the rest. chatId: z.string().min(1).optional(), @@ -75,10 +90,17 @@ const ActionBody = z.object({ uri: z.string().optional(), // A JSON array of `trigger://` URIs, for `resolve-many`. uris: z.string().optional(), + // The watch to cancel, for `watch-cancel`. + watchId: z.string().min(1).optional(), + // The configured card, for `watch-create`: a JSON `WatchDraft`. + draft: z.string().optional(), + // Stable per card submission, so a retried `watch-create` repairs instead of repeating. + // Required for `watch-create`: see the check in that branch. + clientRequestId: z.string().min(1).max(64).optional(), }); // History list by default. `?chatId=` returns the stored transcript plus session, -// `?quota=1` the message count. +// `?unread=1` the unread wake count and recent wakes, `?quota=1` the message count. export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const userId = user.id; @@ -97,6 +119,28 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const searchParams = new URL(request.url).searchParams; + // The wake poll runs once a minute per open tab, so it reads only the org id it needs + // and asks the agent DB one question. The list is recent deliveries, not unread ones; + // the client dedupes by id. + if (searchParams.get("unread") === "1") { + const scoped = await $replica.project.findFirst({ + where: { + slug: projectParam, + organization: { slug: organizationSlug, members: { some: { userId } } }, + }, + select: { organizationId: true }, + }); + if (!scoped) return json({ error: "Project not found" }, { status: 404 }); + + return json( + await readWatchWakeFeed(dashboardAgentDb, { + organizationId: scoped.organizationId, + userId, + deliveredAfter: new Date(Date.now() - 15 * 60 * 1000), + }) + ); + } + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) return json({ error: "Project not found" }, { status: 404 }); @@ -125,17 +169,40 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { userId, }); - // One query for all the listed chats, never one per row. - const investigatingChatIds = await listChatIdsWithOpenInvestigations(dashboardAgentDb, { - organizationId: project.organizationId, - userId, - }); + // One query each for all the listed chats, never one per row. + const [watchesByChat, unreadWakes, unreadChatIds, investigatingChatIds] = await Promise.all([ + listActiveWatchesForChats({ + chatIds: chats.map((chat) => chat.id), + organizationId: project.organizationId, + userId, + }), + countUnreadWatchWakes(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + listChatIdsWithUnreadWakes(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + listChatIdsWithOpenInvestigations(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + ]); return json({ - chats: chats.map((chat) => ({ - ...chat, - hasOpenInvestigation: investigatingChatIds.has(chat.id), - })), + chats: chats.map((chat) => { + const watches = watchesByChat[chat.id] ?? []; + return { + ...chat, + watches, + hasUnreadWake: unreadChatIds.has(chat.id), + // `watches` also carries fired and expired rows, so check for active here. + hasActiveWatch: watches.some((watch) => watch.status === "active"), + hasOpenInvestigation: investigatingChatIds.has(chat.id), + }; + }), + unreadWakes, }); }; @@ -350,6 +417,91 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return json({ resolved }); } + // The configuration card's submit path. The environment comes from the URL and goes + // through the same re-authorization a background tick passes, never from the body. + if (parsed.data.intent === "watch-create") { + // No fallback: a per-condition key would identify the condition rather than this + // submit, so a re-watch could replay a stale terminal outcome. + const clientRequestId = parsed.data.clientRequestId; + if (!clientRequestId) { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + + let draft: WatchDraft; + try { + const result = watchDraftSchema.safeParse(JSON.parse(parsed.data.draft ?? "")); + if (!result.success) { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + draft = result.data; + } catch { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + + const runtimeEnv = await findEnvironmentBySlug(project.id, envParam, userId); + if (!runtimeEnv) return json({ error: "Environment not found" }, { status: 404 }); + + const environment = await authorizeWatchEnvironmentById({ + userId, + environmentId: runtimeEnv.id, + }); + if (!environment) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + + // A watch is chat-bound, so a card submitted from a fresh panel creates a chat. + const targetChatId = parsed.data.chatId; + if ( + targetChatId && + !(await chatExists(dashboardAgentDb, { + chatId: targetChatId, + userId, + organizationId: project.organizationId, + })) + ) { + return json({ error: "Chat not found", code: "chat_not_found" }, { status: 404 }); + } + + // The request record is written before the watch and the confirmation after, so a + // half-finished submit is repairable and never leaves a watch nobody can see. + const result = await submitDashboardAgentWatch({ + environment, + userId, + organizationId: project.organizationId, + chatId: targetChatId, + clientRequestId, + draft, + }); + + if (!result.ok) { + const status = + result.code === "limit_reached" || + result.code === "duplicate" || + result.code === "request_conflict" + ? 409 + : result.code === "invalid_target" || result.code === "chat_not_found" + ? 404 + : result.code === "not_configured" + ? 501 + : 500; + return json( + { + error: result.error, + code: result.code, + ...(result.existingId ? { existingId: result.existingId } : {}), + }, + { status } + ); + } + + return json({ + chatId: result.chatId, + watching: result.watching, + watchId: result.watchId, + messages: result.messages, + }); + } + const { intent, chatId } = parsed.data; if (!chatId) return json({ error: "chatId is required" }, { status: 400 }); @@ -426,9 +578,19 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return json({ ok: true }); } + // The update is owner-scoped, so a chatId the caller doesn't own is a no-op. + case "read": { + await markChatRead(dashboardAgentDb, { + chatId, + userId, + organizationId: project.organizationId, + }); + return json({ ok: true }); + } + case "delete": { - // `softDeleteChat` is owner-scoped but takes no org, so the org scope has to be - // enforced here. + // `deleteChatWithWatches` is owner-scoped but takes no org, so the org scope has + // to be enforced here. if ( !(await chatExists(dashboardAgentDb, { chatId, @@ -438,7 +600,35 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { ) { return json({ error: "Chat not found" }, { status: 404 }); } - await softDeleteChat(dashboardAgentDb, { chatId, userId }); + // The delete and the watch cancellations land in one transaction. + const { cancelledWatches } = await deleteChatWithWatches({ chatId, userId }); + return json({ ok: true, cancelledWatches }); + } + + // Ownership goes through the chat: the watch must belong to the named chat, and + // that chat to this user in this org. + case "watch-cancel": { + const watchId = parsed.data.watchId; + if (!watchId) return json({ error: "watchId is required" }, { status: 400 }); + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch || watch.chatId !== chatId) { + return json({ error: "Watch not found" }, { status: 404 }); + } + + if ( + !(await chatExists(dashboardAgentDb, { + chatId, + userId, + organizationId: project.organizationId, + })) + ) { + return json({ error: "Chat not found" }, { status: 404 }); + } + + // `cancelWatch` only touches an active row, so an already-resolved watch keeps + // its outcome and this is a no-op. + await cancelWatch(dashboardAgentDb, { id: watchId, reason: "user" }); return json({ ok: true }); } } diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 288a0c27a84..374440dfa0b 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -33,6 +33,9 @@ import { MachineLabelCombo } from "~/components/MachineLabelCombo"; import { MachineTooltipInfo } from "~/components/MachineTooltipInfo"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { InvestigateButton } from "~/components/dashboard-agent/InvestigateButton"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { isFinalRunStatus } from "~/v3/taskStatus"; +import { runWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; import { failedRunPrompt, isFailedRunStatus, @@ -1147,6 +1150,16 @@ function RunBody({ runFriendlyId={run.friendlyId} /> ) : null} + {/* The universal `Watch…` entry (§2.1), pre-filled with this run's + recommendation: tell me when it finishes. Only while the run can + still change — a finished run has nothing left to wait for. */} + {isFinalRunStatus(run.status) ? null : ( + + )} {run.error && ( diff --git a/apps/webapp/app/services/dashboardAgentAlertContext.server.ts b/apps/webapp/app/services/dashboardAgentAlertContext.server.ts new file mode 100644 index 00000000000..7266991265f --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentAlertContext.server.ts @@ -0,0 +1,57 @@ +/** + * From the turn's environment scope and a chat id to an authorized environment. Same order + * of authority as the watches route: token environment, chat ownership, re-authorization. + */ + +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + authorizeWatchEnvironmentById, + resolveChatWatchContext, +} from "~/services/dashboardAgentWatches.server"; + +export type AgentAlertContextError = "chat_not_found" | "invalid_target" | "environment_mismatch"; + +export type AgentAlertContext = + | { ok: true; environment: AuthenticatedEnvironment } + | { ok: false; code: AgentAlertContextError; error: string }; + +export async function resolveAgentAlertContext(params: { + userId: string; + chatId: string; + /** The turn's environment scope, off the user-actor token. The authority here. */ + environmentId: string; + /** Optional echoes from the request body. Checked, never trusted. */ + claimedEnvironmentId?: string; + claimedProjectRef?: string; +}): Promise { + if (params.claimedEnvironmentId && params.claimedEnvironmentId !== params.environmentId) { + return { + ok: false, + code: "environment_mismatch", + error: "That environment isn't the one this chat is open in.", + }; + } + + const chat = await resolveChatWatchContext({ chatId: params.chatId, userId: params.userId }); + if (!chat) { + return { ok: false, code: "chat_not_found", error: "Chat not found" }; + } + + const environment = await authorizeWatchEnvironmentById({ + userId: params.userId, + environmentId: params.environmentId, + }); + if (!environment || environment.organizationId !== chat.organizationId) { + return { ok: false, code: "invalid_target", error: "Environment not found" }; + } + + if (params.claimedProjectRef && environment.project.externalRef !== params.claimedProjectRef) { + return { + ok: false, + code: "environment_mismatch", + error: "That project isn't the one this chat is open in.", + }; + } + + return { ok: true, environment }; +} diff --git a/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts b/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts new file mode 100644 index 00000000000..4c43b962f33 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts @@ -0,0 +1,63 @@ +/** + * The credential in a watch alert email's unsubscribe link. HS256 over `SESSION_SECRET` with + * a prefix and `kind` claim disjoint from every other token signed with that secret. + */ + +import { generateJWT, validateJWT } from "@trigger.dev/core/v3/jwt"; +import { env } from "~/env.server"; + +const UNSUBSCRIBE_TOKEN_PREFIX = "tr_daau_"; +const UNSUBSCRIBE_TOKEN_KIND = "dashboard_agent_alert_unsubscribe"; +const UNSUBSCRIBE_PURPOSE = "unsubscribe"; + +/** Long-lived: an alert email has to keep working months after it arrived. */ +const UNSUBSCRIBE_TOKEN_TTL = "365d"; + +export type UnsubscribeTokenClaims = { channelId: string; alertType: string }; + +export async function signDashboardAgentAlertUnsubscribeToken( + secret: string, + opts: { channelId: string; alertType: string } +): Promise { + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: UNSUBSCRIBE_TOKEN_KIND, + purpose: UNSUBSCRIBE_PURPOSE, + sub: opts.channelId, + alertType: opts.alertType, + }, + expirationTime: UNSUBSCRIBE_TOKEN_TTL, + }); + + return `${UNSUBSCRIBE_TOKEN_PREFIX}${jwt}`; +} + +export async function verifyDashboardAgentAlertUnsubscribeToken( + secret: string, + token: string +): Promise { + if (!token.startsWith(UNSUBSCRIBE_TOKEN_PREFIX)) return; + + const result = await validateJWT(token.slice(UNSUBSCRIBE_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== UNSUBSCRIBE_TOKEN_KIND) return; + if (payload.purpose !== UNSUBSCRIBE_PURPOSE) return; + if (typeof payload.sub !== "string" || payload.sub.length === 0) return; + if (typeof payload.alertType !== "string" || payload.alertType.length === 0) return; + + return { channelId: payload.sub, alertType: payload.alertType }; +} + +export function mintDashboardAgentAlertUnsubscribeToken(opts: { + channelId: string; + alertType: string; +}): Promise { + return signDashboardAgentAlertUnsubscribeToken(env.SESSION_SECRET, opts); +} + +export function verifyUnsubscribeToken(token: string): Promise { + return verifyDashboardAgentAlertUnsubscribeToken(env.SESSION_SECRET, token); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts b/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts new file mode 100644 index 00000000000..7cc4e6feb2c --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts @@ -0,0 +1,291 @@ +/** + * The seam between a watch firing and the standard alert pipeline: the enqueue, plus the + * gate both the fan-out and the agent's subscribe endpoint consult. + */ + +import { type Watch } from "@internal/dashboard-agent-db"; +import { type PrismaClientOrTransaction } from "@trigger.dev/database"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { alertsWorker } from "~/v3/alertsWorker.server"; +import { canAccessDashboardAgent } from "~/v3/canAccessDashboardAgent.server"; +import { CreateAlertChannelService } from "~/v3/services/alerts/createAlertChannel.server"; + +/** The alert type a watch fires under. */ +export const DASHBOARD_AGENT_WATCH_ALERT_TYPE = "DASHBOARD_AGENT_WATCH" as const; + +/** What the enqueue needs off a watch row, rather than the full row. */ +export type WatchFiredAlertSource = Pick< + Watch, + | "id" + | "identity" + | "spec" + | "organizationId" + | "projectId" + | "environmentId" + | "userId" + | "firedAt" + | "lastResult" + | "resolution" + | "observedOutcome" +>; + +/** + * Queue the alert fan-out for a resolved watch. Only `fired` dispatches; an expiry is + * narrated in the chat. The job id is the idempotency key: one fan-out per watch. + */ +export async function enqueueWatchFiredAlert( + watch: WatchFiredAlertSource, + outcome: "fired" | "expired" +): Promise { + if (outcome !== "fired") return; + + await alertsWorker.enqueue({ + id: `watch-alert:${watch.id}`, + job: "v3.deliverDashboardAgentWatchAlert", + payload: { + watchId: watch.id, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + userId: watch.userId, + identity: watch.identity, + kind: watch.spec.kind, + note: watch.spec.note, + firedAt: (watch.firedAt ?? new Date()).toISOString(), + facts: watch.lastResult ?? {}, + // The frozen resolved result: the email renders from these and never re-reads + // the source. + resolution: watch.resolution ?? "condition_met", + observed: watch.observedOutcome ?? undefined, + }, + }); +} + +export type DashboardAgentAlertDenyReason = + /** The user can't use the dashboard agent, so its watches can't alert either. */ + | "dashboard_agent_disabled" + /** This installation has no alert email transport configured. */ + | "email_alerts_not_configured"; + +export type DashboardAgentAlertGate = + | { allowed: true } + | { allowed: false; reason: DashboardAgentAlertDenyReason }; + +/** + * May this user's watches alert at all? Operational checks only, no plan check: billing + * gates that separately. `organizationId` stays in the signature for that gate. + */ +export async function canUseDashboardAgentAlerts(params: { + userId: string; + organizationSlug: string; + organizationId: string; + isAdmin?: boolean; + orgFeatureFlags?: Record | null; +}): Promise { + const hasAgent = await canAccessDashboardAgent({ + userId: params.userId, + isAdmin: params.isAdmin ?? false, + // Never an impersonated session: this runs in the background. + isImpersonating: false, + organizationSlug: params.organizationSlug, + orgFeatureFlags: params.orgFeatureFlags, + }); + if (!hasAgent) return { allowed: false, reason: "dashboard_agent_disabled" }; + + return { allowed: true }; +} + +/** + * Whether a fired watch in this environment would already reach this user outside the + * chat. Advisory only: the watch already exists, so every failure answers `none`. + */ +export async function resolveWatchEmailAlertsState(params: { + userId: string; + environment: AuthenticatedEnvironment; +}): Promise<"subscribed" | "none" | "unavailable"> { + const { userId, environment } = params; + try { + // Another member's channel mails them, not this user, so only this user's own channel + // answers "subscribed". + const owner = await resolveWatchAlertOwnership(userId, $replica); + const channel = owner + ? await $replica.projectAlertChannel.findFirst({ + where: { + projectId: environment.project.id, + deduplicationKey: owner.deduplicationKey, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + environmentTypes: { has: environment.type }, + }, + select: { id: true }, + }) + : null; + if (channel) return "subscribed"; + + const gate = await canUseDashboardAgentAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + orgFeatureFlags: environment.organization.featureFlags as Record | null, + }); + return gate.allowed ? "none" : "unavailable"; + } catch (error) { + logger.error("Failed to resolve dashboard agent watch alert state", { + error, + userId, + organizationId: environment.organizationId, + projectId: environment.project.id, + environmentId: environment.id, + }); + return "none"; + } +} + +/** The same gate plus an email transport, or the channel would never deliver. */ +export async function canUseDashboardAgentEmailAlerts( + params: Parameters[0] & { projectId: string } +): Promise { + const base = await canUseDashboardAgentAlerts(params); + if (!base.allowed) return base; + + // Mirrors what the alerts email client needs, not resend specifically. + if (env.ALERT_FROM_EMAIL === undefined || env.ALERT_EMAIL_TRANSPORT === undefined) { + return { allowed: false, reason: "email_alerts_not_configured" }; + } + + return { allowed: true }; +} + +// A channel has no owner column, so this key is the only record of whose channel it is. +export function watchAlertDeduplicationKey(email: string): string { + return `dashboard-agent-watch:${email}`; +} + +/** + * The one place a user id becomes watch-alert ownership. Reading state, subscribing and + * unsubscribing all go through here, so they cannot disagree about whose channel is whose. + */ +async function resolveWatchAlertOwnership( + userId: string, + db: PrismaClientOrTransaction = prisma +): Promise<{ email: string; deduplicationKey: string } | undefined> { + const user = await db.user.findFirst({ where: { id: userId }, select: { email: true } }); + if (!user) return undefined; + return { email: user.email, deduplicationKey: watchAlertDeduplicationKey(user.email) }; +} + +export type SubscribeToWatchAlertsResult = + | { ok: true; email: string } + | { ok: false; reason: DashboardAgentAlertDenyReason | "user_not_found" }; + +/** + * Subscribe the signed-in user's own account email to this project's watch alerts. The + * address is never taken from the request, and the dedup key is stable per (email, project). + */ +export async function subscribeUserToWatchAlerts(params: { + userId: string; + environment: { + type: string; + organizationId: string; + organization: { slug: string }; + project: { id: string; externalRef: string }; + }; +}): Promise { + const { userId, environment } = params; + + const gate = await canUseDashboardAgentEmailAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + projectId: environment.project.id, + }); + if (!gate.allowed) return { ok: false, reason: gate.reason }; + + const owner = await resolveWatchAlertOwnership(userId); + if (!owner) return { ok: false, reason: "user_not_found" }; + + const service = new CreateAlertChannelService(); + await service.call(environment.project.externalRef, userId, { + name: `Watch alerts for ${owner.email}`, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE], + environmentTypes: [environment.type as never], + deduplicationKey: owner.deduplicationKey, + channel: { type: "EMAIL", email: owner.email }, + }); + + return { ok: true, email: owner.email }; +} + +export type UnsubscribeResult = + | { ok: true; channelName: string; disabledChannel: boolean } + | { ok: false; reason: "not_found" | "conflict" }; + +/** How many times a lost race is retried before the caller is told to try again. */ +const UNSUBSCRIBE_ATTEMPTS = 3; + +/** + * Take `DASHBOARD_AGENT_WATCH` off a channel, disabling one left with no alert types. The + * write is conditional on the list the read saw, so a concurrent edit fails this attempt. + * + * A project is shared by every member, so a request-driven caller must pass + * `organizationId` and `ownerUserId` too. + */ +export async function unsubscribeChannelFromWatchAlerts( + channelId: string, + options: { projectId?: string; organizationId?: string; ownerUserId?: string } = {}, + db: PrismaClientOrTransaction = prisma +): Promise { + let ownerKey: string | undefined; + if (options.ownerUserId) { + const owner = await resolveWatchAlertOwnership(options.ownerUserId, db); + if (!owner) return { ok: false, reason: "not_found" }; + ownerKey = owner.deduplicationKey; + } + + const scope = { + id: channelId, + ...(options.projectId ? { projectId: options.projectId } : {}), + ...(options.organizationId ? { project: { organizationId: options.organizationId } } : {}), + ...(ownerKey ? { deduplicationKey: ownerKey } : {}), + }; + + for (let attempt = 0; attempt < UNSUBSCRIBE_ATTEMPTS; attempt++) { + const channel = await db.projectAlertChannel.findFirst({ + where: scope, + select: { name: true, alertTypes: true, projectId: true, deduplicationKey: true }, + }); + // A channel this alert type was never on is out of scope: stripping nothing off it + // would still report success, and an empty list would disable it. + if (!channel || !channel.alertTypes.includes(DASHBOARD_AGENT_WATCH_ALERT_TYPE)) { + return { ok: false, reason: "not_found" }; + } + + const remaining = channel.alertTypes.filter( + (type) => type !== DASHBOARD_AGENT_WATCH_ALERT_TYPE + ); + + const { count } = await db.projectAlertChannel.updateMany({ + // Compare-and-swap on the row the scoped read returned. `updateMany` takes no relation + // filter, so the org scope is carried by the read's `projectId`. + where: { + id: channelId, + projectId: channel.projectId, + deduplicationKey: channel.deduplicationKey, + alertTypes: { equals: channel.alertTypes }, + }, + data: { + alertTypes: remaining, + ...(remaining.length === 0 ? { enabled: false } : {}), + }, + }); + + if (count > 0) { + return { ok: true, channelName: channel.name, disabledChannel: remaining.length === 0 }; + } + } + + return { ok: false, reason: "conflict" }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts b/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts new file mode 100644 index 00000000000..6f7b071d12b --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts @@ -0,0 +1,304 @@ +/** + * The batch check: every due watch of one (environment, cadence) group in one pass. Each row is + * the authority on its own snapshot, and each initiating user is re-authorized before any read. + */ + +import { + cancelWatch, + claimWatchBatchTick, + listActiveWatchesForBatch, + listWatchesAwaitingDeliveryForBatch, + recordWatchAttempt, + recordWatchCheck, + stopWatchBatch, + WATCH_DELIVERY_CLAIM_STALE_MS, + type Watch, +} from "@internal/dashboard-agent-db"; +import type { WatchBatchCheckEntry, WatchBatchCheckResponse } from "@internal/dashboard-agent"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { + checkWatch, + previousCheckFacts, + type WatchCheckDeps, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { + authorizeWatchEnvironment, + type WatchAuthorization, +} from "~/services/dashboardAgentWatches.server"; +import { + mintDashboardAgentWatchToken, + WATCH_TOKEN_GRACE_MS, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * How early a watch may be checked and still count as due, so a tick landing seconds + * early doesn't defer it a whole cadence. Capped at half a cadence. + */ +function dueSlackMs(cadenceMinutes: number): number { + return Math.min(30_000, (cadenceMinutes * 60_000) / 2); +} + +/** Small on purpose: it stops one slow condition serializing the group, not to fan out. */ +const EVALUATION_CONCURRENCY = 8; + +export type WatchBatchCheckDeps = { + now?: () => Date; + /** The group's active watches. */ + listActive?: (params: { environmentId: string; cadenceMinutes: number }) => Promise; + /** The group's resolved watches whose wake is still owed. */ + listOwed?: (params: { + environmentId: string; + cadenceMinutes: number; + claimStaleBefore: Date; + }) => Promise; + /** Re-authorization of one watch's initiating user. */ + authorize?: (watch: Watch) => Promise; + /** The environment readers the conditions run against. */ + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + /** The per-watch token the fired / investigate callbacks are made with. */ + mintToken?: (watch: Watch) => Promise; + concurrency?: number; +}; + +/** + * Run one batch tick's checks. The claim decides whether this run owns the tick and keeps the + * schedule single-file; the guarded transition and fenced delivery claim stop a double fire. + */ +export async function runWatchBatchCheck( + params: { environmentId: string; cadenceMinutes: number; epoch: number; tick: number }, + deps: WatchBatchCheckDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const listActive = + deps.listActive ?? ((args) => listActiveWatchesForBatch(dashboardAgentDb, args)); + const listOwed = + deps.listOwed ?? ((args) => listWatchesAwaitingDeliveryForBatch(dashboardAgentDb, args)); + const mintToken = + deps.mintToken ?? + ((watch: Watch) => + mintDashboardAgentWatchToken({ watchId: watch.id, expiresAt: watch.expiresAt })); + + const claimed = await claimWatchBatchTick(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: params.epoch, + // The tick a run carries is the generation it owns. + generation: params.tick, + }); + if (!claimed) { + logger.debug("Dashboard agent watch batch: the tick is stale", params); + return { stale: true }; + } + + const active = await listActive({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + + const due = active.filter((watch) => isDue(watch, params.cadenceMinutes, now)); + const evaluated = await evaluateGroup(due, params, { ...deps, now: () => now }, mintToken); + + // Wakes this group still owes. Read after the evaluation, so a wake this tick resolved + // and failed to deliver is already in it. + const owed = await listOwed({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + claimStaleBefore: new Date(now.getTime() - WATCH_DELIVERY_CLAIM_STALE_MS), + }); + + const deliveries = await Promise.all( + owed.map(async (watch) => ({ + watchId: watch.id, + token: await mintToken(watch), + // A delivery decides nothing, so it claims no generation. + tick: 0, + deliverOnly: true as const, + })) + ); + + // The chain only stops with nothing to poll and nothing owed: stopping while a wake is + // owed strands it. Fenced on the epoch, so it can only end this run's own chain. + const continues = active.length > 0 || owed.length > 0; + if (!continues) { + await stopWatchBatch(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: params.epoch, + }); + } + + return { watches: [...evaluated, ...deliveries], continues }; +} + +/** + * A watch whose window closes before the next tick is due now, so its final evaluation is + * never missed. A watch past the token grace is never due: the expiry sweep owns it. + */ +export function isDue(watch: Watch, cadenceMinutes: number, now: Date): boolean { + const nowMs = now.getTime(); + const cadenceMs = cadenceMinutes * 60_000; + + if (nowMs > watch.expiresAt.getTime() + WATCH_TOKEN_GRACE_MS) return false; + if (watch.expiresAt.getTime() <= nowMs + cadenceMs) return true; + + const lastChecked = watch.lastCheckedAt?.getTime(); + return lastChecked === undefined || lastChecked <= nowMs - cadenceMs + dueSlackMs(cadenceMinutes); +} + +/** + * Evaluate the due watches against one set of readers. Authorization is cached per (user, org, + * project); each watch runs in its own try, so a failure is that watch's answer alone. + */ +async function evaluateGroup( + due: Watch[], + params: { environmentId: string; cadenceMinutes: number }, + deps: WatchBatchCheckDeps, + mintToken: (watch: Watch) => Promise +): Promise { + if (due.length === 0) return []; + + const now = deps.now?.() ?? new Date(); + const authorize = deps.authorize ?? defaultAuthorize; + const buildCheckDeps = deps.checkDeps ?? watchCheckDeps; + + const authorizations = new Map>(); + const authorizeOnce = (watch: Watch) => { + const key = `${watch.userId}:${watch.organizationId}:${watch.projectId}`; + const cached = authorizations.get(key); + if (cached) return cached; + const pending = authorize(watch); + authorizations.set(key, pending); + return pending; + }; + + // Built from the first authorization that passes, then shared: every row in the group + // names the same environment. + let readers: WatchCheckDeps | undefined; + + const evaluateOne = async ( + watch: Watch, + base: { watchId: string; token: string; tick: number } + ): Promise => { + const authorization = await authorizeOnce(watch); + if (!authorization.ok) { + // Cancel before anything is read: a watch must not outlive its creator's access. + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "access_revoked" }); + return { ...base, code: "access_revoked", error: "Access to this environment was revoked" }; + } + + readers ??= shareReads(buildCheckDeps(authorization.environment, now)); + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const final = watch.expiresAt.getTime() <= now.getTime(); + const outcome = await checkWatch( + watch.spec, + readers, + // A check that couldn't read anything freezes a streak instead of resetting it. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch batch: a check failed", { + error, + watchId: watch.id, + environmentId: params.environmentId, + }) + ); + + // Only a real evaluation is recorded, final or not: `unavailable` means nothing was read, + // so writing it would move `lastCheckedAt` and overwrite the facts a streak lives in. + // Guarded on `active`, and never touches `tickCount`. + if (outcome.result !== "unavailable") { + await recordWatchCheck(dashboardAgentDb, { + id: watch.id, + lastResult: { + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + final, + }, + }); + } else { + // Looked at, not checked: this rotates the watch out of its group's head without + // touching its dueness or the facts its streak lives in. + await recordWatchAttempt(dashboardAgentDb, { id: watch.id }); + } + + return { ...base, result: outcome.result, facts: outcome.facts, observed: outcome.observed }; + }; + + return mapWithConcurrency(due, deps.concurrency ?? EVALUATION_CONCURRENCY, async (watch) => { + // Minted outside the try, because the catch below needs a token it can't fail to have. + const base = { watchId: watch.id, token: await mintToken(watch), tick: watch.tickCount + 1 }; + try { + return await evaluateOne(watch, base); + } catch (error) { + logger.error("Dashboard agent watch batch: a watch couldn't be evaluated", { + watchId: watch.id, + environmentId: params.environmentId, + error, + }); + // `unavailable` is never read as true or false: the watch keeps its state. Still a + // look, so the fairness key moves even when nothing else does. + await recordWatchAttempt(dashboardAgentDb, { id: watch.id }).catch(() => {}); + return { ...base, result: "unavailable" as const, error: (error as Error).message }; + } + }); +} + +/** + * Wrap a batch's readers so each distinct read happens once. `now` is fixed for the batch, so + * a reader's answer is a pure function of its arguments. Failed reads are cached too. + * + * Exported for the expiry sweep, which finalizes the same rows against the same readers. + */ +export function shareReads(readers: WatchCheckDeps): WatchCheckDeps { + const cache = new Map>(); + const once = (name: string, read: (...args: A) => Promise) => { + return (...args: A): Promise => { + const key = `${name}:${JSON.stringify(args)}`; + const cached = cache.get(key); + if (cached) return cached as Promise; + const pending = read(...args); + cache.set(key, pending); + return pending; + }; + }; + + return { + readRun: once("readRun", readers.readRun), + queueExists: once("queueExists", readers.queueExists), + readQueueDepth: once("readQueueDepth", readers.readQueueDepth), + readQueueOldestAge: once("readQueueOldestAge", readers.readQueueOldestAge), + readErrorRecurrence: once("readErrorRecurrence", readers.readErrorRecurrence), + readHealth: once("readHealth", readers.readHealth), + }; +} + +/** `mapper` over `items`, at most `limit` in flight. Order is preserved. */ +export async function mapWithConcurrency( + items: T[], + limit: number, + mapper: (item: T) => Promise +): Promise { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(Math.max(limit, 1), items.length) }, async () => { + while (next < items.length) { + const index = next++; + results[index] = await mapper(items[index]!); + } + }); + await Promise.all(workers); + return results; +} + +function defaultAuthorize(watch: Watch): Promise { + return authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts b/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts new file mode 100644 index 00000000000..fc41bcba038 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts @@ -0,0 +1,106 @@ +/** + * The reader contract every watch condition family shares, plus the duration formatting + * they all label with. No IO of its own. + */ + +import { formatDurationMilliseconds } from "@trigger.dev/core/v3/utils/durations"; +import type { WatchCheckResult, WatchObservedOutcome } from "@internal/dashboard-agent-contracts"; + +/** The single run point-read. Postgres is authoritative for run state. */ +export type WatchRunRow = { + friendlyId: string; + status: string; + queue: string; + createdAt: Date; + /** Stamped when the run entered the queue. NULL while a run is delayed. */ + queuedAt: Date | null; + /** Set once the run is dequeued. */ + startedAt: Date | null; + completedAt: Date | null; + delayUntil: Date | null; +}; + +export type WatchQueueDepth = { + /** Pending count for the queue, as of `asOf`. */ + depth: number; + source: "live_queue" | "queue_metrics"; + /** A stale reading can never answer "drained". */ + current: boolean; + /** What instant the reading describes, when it isn't the live counter. */ + asOf?: Date; +}; + +/** + * The oldest still-waiting run's age in one queue. A non-current age is wrong in both + * directions, so `checkQueueOldestAge` refuses it rather than comparing it. + */ +export type WatchQueueOldestAge = { + /** Age of the oldest run still waiting, in ms. Null when nothing is waiting. */ + ageMs: number | null; + source: "live_queue" | "queue_metrics"; + current: boolean; + asOf?: Date; +}; + +/** What we know about the watched error's occurrences relative to `since`. */ +export type WatchErrorRecurrence = { + /** Earliest occurrence proven after `since`. Null with a `lastSeenAt` means not since. */ + occurredAt: Date | null; + /** How precisely `occurredAt` is known: to the millisecond, or to its minute. */ + occurredAtPrecision: "exact" | "minute" | null; + /** Occurrences after `since`. A lower bound when `countApproximate`. */ + countSince: number; + /** True when occurrences in the watch's creation minute can't be separated out. */ + countApproximate: boolean; + /** The fingerprint's most recent occurrence, whenever it was. */ + lastSeenAt: Date | null; +}; + +export type WatchHealthSeverity = "ok" | "warn" | "crit"; + +export type WatchHealthSnapshot = { + /** `facts.trustworthy` from the health report. Untrustworthy never fires recovery. */ + trustworthy: boolean; + severity: WatchHealthSeverity; +}; + +/** + * The readers a check may use. Each may throw, which the caller turns into `unavailable`. + * `null` means the source answered and there is nothing there. + */ +export type WatchCheckDeps = { + /** Run point-read by public run id, scoped to the watch's environment. */ + readRun: (runId: string) => Promise; + /** Does this queue exist in the watch's environment? */ + queueExists: (queue: string) => Promise; + /** Current pending count, live run-queue first with a ClickHouse fallback. */ + readQueueDepth: (queue: string) => Promise; + /** Age of the oldest run still waiting in the queue, right now. */ + readQueueOldestAge: (queue: string) => Promise; + /** `null` means the fingerprint has no occurrences at all in this environment. */ + readErrorRecurrence: (fingerprint: string, since: Date) => Promise; + /** The health report's current verdict for the watch's environment. */ + readHealth: () => Promise; +}; + +export type WatchCheckInput = { + now: Date; + /** The recurrence window's start: the server-set `spec.since`, never caller-set. */ + since: Date; + /** + * The previous check's facts, for the stateful kinds. A check's own facts are the only + * storage for its state. Absent means no prior observation, never zero. + */ + previous?: Record | null; +}; + +export type WatchCheckOutcome = { + result: WatchCheckResult; + facts: Record; + /** Frozen onto the row by the resolving transition, so no surface re-reads the source. */ + observed: WatchObservedOutcome; +}; + +export function formatMs(ms: number): string { + return formatDurationMilliseconds(ms, { style: "short", maxDecimalPoints: 0 }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts b/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts new file mode 100644 index 00000000000..caa2cc1340d --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts @@ -0,0 +1,331 @@ +/** + * Default IO wiring for the watch checks. Run state and queue existence are authoritative + * Postgres point-reads, and readers throw rather than invent a zero on a broken source. + */ + +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { $replica, prisma } from "~/db.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { ReportPresenter } from "~/presenters/v3/reports/ReportPresenter.server"; +import { engine } from "~/v3/runEngine.server"; +import { runStore } from "~/v3/runStore.server"; +import type { + WatchCheckDeps, + WatchErrorRecurrence, + WatchHealthSeverity, + WatchHealthSnapshot, + WatchQueueDepth, + WatchQueueOldestAge, + WatchRunRow, +} from "./dashboardAgentWatchChecks"; + +const WATCH_RUN_SELECT = { + friendlyId: true, + status: true, + queue: true, + createdAt: true, + queuedAt: true, + startedAt: true, + completedAt: true, + delayUntil: true, +} as const; + +/** The single Postgres point-read: one run, scoped to the watch's environment. */ +export async function readWatchRun( + runFriendlyId: string, + environmentId: string +): Promise { + const run = await runStore.findRun( + { friendlyId: runFriendlyId, runtimeEnvironmentId: environmentId }, + { select: WATCH_RUN_SELECT }, + $replica + ); + return run ?? null; +} + +/** One Postgres point-read: does this queue exist in the environment? */ +export async function watchQueueExists(environmentId: string, queueName: string): Promise { + const queue = await $replica.taskQueue.findFirst({ + where: { runtimeEnvironmentId: environmentId, name: queueName }, + select: { id: true }, + }); + return queue !== null; +} + +/** The same run read on the primary, for a target that may have been created a moment ago. */ +export async function readWatchRunOnPrimary( + runFriendlyId: string, + environmentId: string +): Promise { + const run = await runStore.findRunOnPrimary( + { friendlyId: runFriendlyId, runtimeEnvironmentId: environmentId }, + { select: WATCH_RUN_SELECT } + ); + return run ?? null; +} + +/** The same queue read on the primary. */ +export async function watchQueueExistsOnPrimary( + environmentId: string, + queueName: string +): Promise { + const queue = await prisma.taskQueue.findFirst({ + where: { runtimeEnvironmentId: environmentId, name: queueName }, + select: { id: true }, + }); + return queue !== null; +} + +/** How far back the ClickHouse depth fallback looks when the live counter is down. */ +const DEPTH_FALLBACK_MINUTES = 10; +const DEPTH_FALLBACK_BUCKET_SECONDS = 60; +/** + * How far behind `now` the newest analytics bucket may end and still count as current. + * One bucket of slack: anything older leaves runs queued in the gap invisible. + */ +const DEPTH_FRESH_TOLERANCE_MS = DEPTH_FALLBACK_BUCKET_SECONDS * 1000; + +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().slice(0, 19).replace("T", " "); +} + +/** ClickHouse renders DateTime without a zone; the column is UTC. */ +function parseClickhouseDateTime(value: string): Date { + return new Date(`${value.replace(" ", "T")}Z`); +} + +/** + * Current pending count for one queue. The live counter is the truth; the ClickHouse fallback + * reports the newest bucket's peak depth and is `current` only if it reaches the present. + */ +export async function readWatchQueueDepth( + environment: AuthenticatedEnvironment, + queueName: string, + now: Date = new Date() +): Promise { + const live = await engine.lengthOfQueue(environment, queueName).catch(() => null); + if (typeof live === "number" && Number.isFinite(live)) { + return { depth: live, source: "live_queue", current: true, asOf: now }; + } + + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + environment.organizationId, + "query" + ); + + const bucketMs = DEPTH_FALLBACK_BUCKET_SECONDS * 1000; + const endMs = Math.ceil(now.getTime() / bucketMs) * bucketMs; + const startMs = endMs - DEPTH_FALLBACK_MINUTES * 60_000; + + const [error, rows] = await clickhouse.queueMetrics.depthSparklines({ + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: environment.id, + queueNames: [queueName], + startTime: formatClickhouseDateTime(new Date(startMs)), + endTime: formatClickhouseDateTime(new Date(endMs)), + bucketSeconds: DEPTH_FALLBACK_BUCKET_SECONDS, + }); + + if (error) throw error; + if (!rows || rows.length === 0) return null; + + // Newest bucket wins: the closest the rollup gets to now. + const newest = rows.reduce((best, row) => (row.bucket > best.bucket ? row : best), rows[0]!); + const bucketEnd = new Date(parseClickhouseDateTime(newest.bucket).getTime() + bucketMs); + const current = bucketEnd.getTime() >= now.getTime() - DEPTH_FRESH_TOLERANCE_MS; + + return { depth: newest.depth, source: "queue_metrics", current, asOf: bucketEnd }; +} + +/** + * How long the oldest still-waiting run in a queue has waited: for a concurrency-keyed queue the + * worst across keys with a live backlog. `null` can't be read, `ageMs: null` is empty. + */ +export async function readWatchQueueOldestAge( + environment: AuthenticatedEnvironment, + queueName: string, + now: Date = new Date() +): Promise { + const [breakdown, oldestQueuedAt] = await Promise.all([ + engine.concurrencyKeyBreakdown(environment, queueName, { limit: OLDEST_AGE_CK_LIMIT }), + engine.oldestMessageInQueue(environment, queueName), + ]); + + const waitingKeys = breakdown.keys.filter((key) => key.queued > 0); + const ageMs = + waitingKeys.length > 0 + ? waitingKeys.reduce((max, key) => Math.max(max, now.getTime() - key.oldestEnqueuedAt), 0) + : typeof oldestQueuedAt === "number" + ? Math.max(0, now.getTime() - oldestQueuedAt) + : null; + + return { ageMs, source: "live_queue", current: true, asOf: now }; +} + +/** Same cap the queue detail page reads keys with. */ +const OLDEST_AGE_CK_LIMIT = 50; + +const MINUTE_MS = 60_000; + +type OrganizationClickhouse = Awaited< + ReturnType +>; + +/** + * The fingerprint's most recent occurrence at millisecond precision, from `errors_v1`. The + * per-minute rollup can't separate the prompting error from a recurrence in the same minute. + */ +async function readErrorLastSeen( + clickhouse: OrganizationClickhouse, + environment: AuthenticatedEnvironment, + fingerprint: string +): Promise { + const builder = clickhouse.errors.activeErrorsSinceQueryBuilder(); + builder.where("organization_id = {organizationId: String}", { + organizationId: environment.organizationId, + }); + builder.where("project_id = {projectId: String}", { projectId: environment.projectId }); + builder.where("environment_id = {environmentId: String}", { environmentId: environment.id }); + builder.where("error_fingerprint = {fingerprint: String}", { fingerprint }); + builder.groupBy("environment_id, task_identifier, error_fingerprint"); + + const [error, rows] = await builder.execute(); + if (error) throw error; + if (!rows || rows.length === 0) return null; + + let lastSeenMs = 0; + for (const row of rows) { + const ms = Number(row.last_seen); + if (Number.isFinite(ms) && ms > lastSeenMs) lastSeenMs = ms; + } + + return lastSeenMs > 0 ? new Date(lastSeenMs) : null; +} + +/** + * What we know about a fingerprint relative to `since`. `errors_v1` decides whether it + * recurred; the rollup's count is a lower bound when the creation-minute bucket has hits. + */ +export async function readWatchErrorRecurrence( + environment: AuthenticatedEnvironment, + fingerprint: string, + since: Date +): Promise { + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + environment.organizationId, + "logs" + ); + + const lastSeenAt = await readErrorLastSeen(clickhouse, environment, fingerprint); + // Never seen in this environment at all. + if (!lastSeenAt) return null; + + const notRecurred: WatchErrorRecurrence = { + occurredAt: null, + occurredAtPrecision: null, + countSince: 0, + countApproximate: false, + lastSeenAt, + }; + if (lastSeenAt.getTime() <= since.getTime()) return notRecurred; + + // Something landed after `since`; the rollup fills in the count and the minute. + const sinceMinuteMs = Math.floor(since.getTime() / MINUTE_MS) * MINUTE_MS; + const queryBuilder = clickhouse.errors.createOccurrencesQueryBuilder("INTERVAL 1 MINUTE"); + queryBuilder.where("organization_id = {organizationId: String}", { + organizationId: environment.organizationId, + }); + queryBuilder.where("project_id = {projectId: String}", { projectId: environment.projectId }); + queryBuilder.where("environment_id = {environmentId: String}", { environmentId: environment.id }); + queryBuilder.where("error_fingerprint = {fingerprint: String}", { fingerprint }); + // The creation minute is included; its occurrences are counted separately below. + queryBuilder.where("minute >= toStartOfMinute(fromUnixTimestamp64Milli({sinceMs: Int64}))", { + sinceMs: since.getTime(), + }); + queryBuilder.groupBy("error_fingerprint, bucket_epoch"); + queryBuilder.orderBy("bucket_epoch ASC"); + + const [error, rows] = await queryBuilder.execute(); + if (error) throw error; + + let earliestAfterMs: number | null = null; + let countAfter = 0; + let creationMinuteCount = 0; + + for (const row of rows ?? []) { + const bucketMs = row.bucket_epoch * 1000; + if (bucketMs <= sinceMinuteMs) { + creationMinuteCount += row.count; + continue; + } + countAfter += row.count; + if (earliestAfterMs === null || bucketMs < earliestAfterMs) earliestAfterMs = bucketMs; + } + + // The earliest provable occurrence: a bucket starting after the creation minute, or the + // exact `last_seen` when that is the only evidence. + const useBucket = earliestAfterMs !== null && earliestAfterMs < lastSeenAt.getTime(); + + return { + occurredAt: useBucket ? new Date(earliestAfterMs!) : lastSeenAt, + occurredAtPrecision: useBucket ? "minute" : "exact", + // At least the one `errors_v1` proved, even if the rollup lags behind it. + countSince: Math.max(1, countAfter), + countApproximate: creationMinuteCount > 0, + lastSeenAt, + }; +} + +const HEALTH_SEVERITIES = new Set(["ok", "warn", "crit"]); + +/** + * The health report's current verdict, from the existing interpreter. No health reasoning + * is re-implemented here. + */ +export async function readWatchHealth( + environment: AuthenticatedEnvironment +): Promise { + const report = await new ReportPresenter().call({ environment, key: "health" }); + if (!report) return null; + + const severity = report.summary.severity; + if (!HEALTH_SEVERITIES.has(severity)) return null; + + const trustworthy = (report.facts as { trustworthy?: unknown } | undefined)?.trustworthy; + return { + // An absent trust marker counts as untrustworthy. + trustworthy: trustworthy === true, + severity: severity as WatchHealthSeverity, + }; +} + +export function watchCheckDeps( + environment: AuthenticatedEnvironment, + now: Date = new Date() +): WatchCheckDeps { + return { + readRun: (runId) => readWatchRun(runId, environment.id), + queueExists: (queue) => watchQueueExists(environment.id, queue), + readQueueDepth: (queue) => readWatchQueueDepth(environment, queue, now), + readQueueOldestAge: (queue) => readWatchQueueOldestAge(environment, queue, now), + readErrorRecurrence: (fingerprint, since) => + readWatchErrorRecurrence(environment, fingerprint, since), + readHealth: () => readWatchHealth(environment), + }; +} + +/** + * Creation-time deps. The target reads go to the primary, so a run or queue created moments + * ago is visible instead of failing as a non-existent target inside the replication window. + */ +export function watchCreationCheckDeps( + environment: AuthenticatedEnvironment, + now: Date = new Date() +): WatchCheckDeps { + return { + ...watchCheckDeps(environment, now), + readRun: (runId) => readWatchRunOnPrimary(runId, environment.id), + queueExists: (queue) => watchQueueExistsOnPrimary(environment.id, queue), + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchChecks.ts b/apps/webapp/app/services/dashboardAgentWatchChecks.ts new file mode 100644 index 00000000000..717d130ef20 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchChecks.ts @@ -0,0 +1,165 @@ +/** + * Deterministic evaluation of one watch condition, with IO behind `WatchCheckDeps`. `unavailable` + * is never a verdict, durations carry their basis, and `observed` is kept apart from the result. + * + * One module per condition family; this one dispatches and owns the failure envelope. + */ + +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; +import { checkRunFailed, checkRunFinished, checkRunStart } from "./dashboardAgentWatchRunChecks"; +import { + checkBacklogDrain, + checkQueueDepthAbove, + checkQueueDepthBelow, + checkQueueOldestAge, + checkQueueStalled, +} from "./dashboardAgentWatchQueueChecks"; +import { checkErrorRecurrence } from "./dashboardAgentWatchErrorChecks"; +import { checkHealthRecovery } from "./dashboardAgentWatchHealthChecks"; + +export type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, + WatchErrorRecurrence, + WatchHealthSeverity, + WatchHealthSnapshot, + WatchQueueDepth, + WatchQueueOldestAge, + WatchRunRow, +} from "./dashboardAgentWatchCheckBase"; +export { + checkRunFailed, + checkRunFinished, + checkRunStart, + describeRunWait, + isTerminalRunStatus, + type WatchWaitBasis, +} from "./dashboardAgentWatchRunChecks"; +export { + checkBacklogDrain, + checkQueueDepthAbove, + checkQueueDepthBelow, + checkQueueOldestAge, + checkQueueStalled, +} from "./dashboardAgentWatchQueueChecks"; +export { checkErrorRecurrence, normalizeErrorFingerprint } from "./dashboardAgentWatchErrorChecks"; +export { checkHealthRecovery } from "./dashboardAgentWatchHealthChecks"; + +/** + * The previous check's facts out of `lastResult`, which holds raw facts, the check endpoint's + * envelope, or the failure wrapper. The wrapper is unwrapped, so a streak survives a gap. + */ +export function previousCheckFacts(lastResult: unknown): Record | null { + if (!lastResult || typeof lastResult !== "object" || Array.isArray(lastResult)) return null; + const record = lastResult as Record; + + if (record.checkFailed === true) return previousCheckFacts(record.previous); + if (record.facts && typeof record.facts === "object" && !Array.isArray(record.facts)) { + return record.facts as Record; + } + return record; +} + +/** The single place a check failure becomes `unavailable`, never a verdict. */ +export async function checkWatch( + spec: WatchSpec, + deps: WatchCheckDeps, + input: WatchCheckInput, + onError?: (error: unknown) => void +): Promise { + try { + switch (spec.kind) { + case "run_start": + return await checkRunStart(spec, deps, input); + case "run_finished": + return await checkRunFinished(spec, deps, input); + case "run_failed": + return await checkRunFailed(spec, deps, input); + case "backlog_drain": + return await checkBacklogDrain(spec, deps, input); + case "queue_depth_above": + return await checkQueueDepthAbove(spec, deps, input); + case "queue_depth_below": + return await checkQueueDepthBelow(spec, deps, input); + case "queue_stalled": + return await checkQueueStalled(spec, deps, input); + case "queue_oldest_age": + return await checkQueueOldestAge(spec, deps, input); + case "error_recurrence": + return await checkErrorRecurrence(spec, deps, input); + case "health_recovery": + return await checkHealthRecovery(spec, deps, input); + default: { + const unreachable: never = spec; + throw new Error(`Unhandled watch kind: ${JSON.stringify(unreachable)}`); + } + } + } catch (error) { + onError?.(error); + return { + result: "unavailable", + facts: { kind: spec.kind, reason: "check_failed" }, + observed: unobservedOutcome(spec), + }; + } +} + +/** + * The observation for a check that couldn't run. `verified: false` means the condition + * couldn't be confirmed, not that it didn't happen. + */ +export function unobservedOutcome(spec: WatchSpec): WatchObservedOutcome { + switch (spec.kind) { + case "run_start": + return { kind: "run_start", verified: false, status: null, started: false }; + case "run_finished": + return { kind: "run_finished", verified: false, finalStatus: null, durationMs: null }; + case "run_failed": + return { kind: "run_failed", verified: false, finalStatus: null, durationMs: null }; + case "backlog_drain": + return { kind: "backlog_drain", verified: false, depth: null }; + case "queue_depth_above": + return { + kind: "queue_depth_above", + verified: false, + depth: null, + threshold: spec.threshold, + }; + case "queue_depth_below": + return { + kind: "queue_depth_below", + verified: false, + depth: null, + threshold: spec.threshold, + }; + case "queue_stalled": + return { + kind: "queue_stalled", + verified: false, + depth: null, + notDecreasingStreak: 0, + ticks: spec.ticks, + }; + case "queue_oldest_age": + return { + kind: "queue_oldest_age", + verified: false, + ageMs: null, + thresholdMinutes: spec.thresholdMinutes, + }; + case "error_recurrence": + return { kind: "error_recurrence", verified: false, countSince: 0 }; + case "health_recovery": + return { kind: "health_recovery", verified: false, severity: null }; + default: { + const unreachable: never = spec; + throw new Error(`Unhandled watch kind: ${JSON.stringify(unreachable)}`); + } + } +} diff --git a/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts b/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts new file mode 100644 index 00000000000..d317426e25e --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts @@ -0,0 +1,72 @@ +/** The error-recurrence condition family. */ + +import { ErrorId } from "@trigger.dev/core/v3/isomorphic"; +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; + +/** + * The model cites the API error id (`error_`) but ClickHouse stores the raw + * fingerprint. Same normalization the errors API route uses. + */ +export function normalizeErrorFingerprint(fingerprint: string): string { + return ErrorId.toId(fingerprint); +} + +/** + * Satisfied on the first occurrence proven to be after the server-set `since`, which is + * never caller-set. The facts carry the precision of what they claim. + */ +export async function checkErrorRecurrence( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const fingerprint = normalizeErrorFingerprint(spec.fingerprint); + const recurrence = await deps.readErrorRecurrence(fingerprint, input.since); + const base = { fingerprint, since: input.since.toISOString() }; + + const quiet: WatchObservedOutcome = { + kind: "error_recurrence", + verified: true, + countSince: 0, + }; + + if (!recurrence) { + return { + result: "pending", + facts: { ...base, countSince: 0, lastSeenAt: null }, + observed: quiet, + }; + } + + const lastSeenAt = recurrence.lastSeenAt?.toISOString() ?? null; + + if (!recurrence.occurredAt) { + return { + result: "pending", + facts: { ...base, countSince: 0, lastSeenAt }, + observed: quiet, + }; + } + + return { + result: "satisfied", + facts: { + ...base, + occurredAt: recurrence.occurredAt.toISOString(), + occurredAtPrecision: recurrence.occurredAtPrecision, + countSince: recurrence.countSince, + countApproximate: recurrence.countApproximate, + lastSeenAt, + }, + observed: { + kind: "error_recurrence", + verified: true, + countSince: recurrence.countSince, + }, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts b/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts new file mode 100644 index 00000000000..443597e5fa9 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts @@ -0,0 +1,49 @@ +/** The health-recovery condition family. */ + +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; + +/** + * Satisfied only when the health report is both trustworthy and `ok`. An untrustworthy + * report can never fire a recovery. + */ +export async function checkHealthRecovery( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const health = await deps.readHealth(); + if (!health) { + return { + result: "unavailable", + facts: { report: spec.report, reason: "report_unavailable" }, + observed: { kind: "health_recovery", verified: false, severity: null }, + }; + } + + const facts = { + report: spec.report, + fromSeverity: spec.fromSeverity, + severity: health.severity, + trustworthy: health.trustworthy, + }; + + if (!health.trustworthy) { + // An untrustworthy report is not an observation of the severity, so record none. + return { + result: "pending", + facts: { ...facts, reason: "untrustworthy" }, + observed: { kind: "health_recovery", verified: false, severity: null }, + }; + } + + return { + result: health.severity === "ok" ? "satisfied" : "pending", + facts, + observed: { kind: "health_recovery", verified: true, severity: health.severity }, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts b/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts new file mode 100644 index 00000000000..7c2011ab04b --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts @@ -0,0 +1,89 @@ +/** + * The reading turn of a consented watch, kicked from here because the wake turn has no token. + * The token is for `watch.userId` against `watch.environmentId`, off the row, never a body. + */ + +import type { WatchInvestigateAction } from "@internal/dashboard-agent"; +import { type Watch } from "@internal/dashboard-agent-db"; +import { watchResultNeedsAttention } from "@internal/dashboard-agent-contracts"; +import { ApiClient } from "@trigger.dev/core/v3"; +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + dashboardAgentApiOrigin, + dashboardAgentEnvironmentName, + mintDashboardAgentUserActorToken, +} from "~/services/dashboardAgent.server"; +import { env } from "~/env.server"; + +/** + * Whether the consent covers this resolved watch. The same contracts call the agent's wake + * makes, so the two can't disagree about which outcomes need attention. + */ +export function watchWantsInvestigation(watch: Watch): boolean { + if (!watch.investigateOnAttention) return false; + if (!watch.resolution) return false; + return watchResultNeedsAttention({ + kind: watch.spec.kind, + resolution: watch.resolution, + outcome: watch.observedOutcome, + }); +} + +/** The action the agent receives. Stable id, so a retried kick is a no-op. */ +export function watchInvestigateAction(watch: Watch): WatchInvestigateAction { + return { + type: "watch.investigate" as const, + id: `watch:${watch.id}:${watch.status}:investigate`, + watchId: watch.id, + identity: watch.identity, + spec: watch.spec, + facts: (watch.lastResult ?? {}) as Record, + resolution: watch.resolution ?? undefined, + observed: watch.observedOutcome ?? undefined, + note: watch.spec.note, + }; +} + +/** + * Send the kick. Throws on failure, and every caller treats it as best-effort: the wake is + * already delivered, so nothing here may retry or invalidate it. + */ +export async function kickWatchInvestigation(params: { + watch: Watch; + environment: AuthenticatedEnvironment; +}): Promise { + const { watch, environment } = params; + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + // The watch's immutable tenancy plus the delegated token that lets the turn read. + const metadata = { + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + projectRef: watch.projectRef ?? environment.project.externalRef, + environmentName: dashboardAgentEnvironmentName(environment.type), + apiOrigin, + userActorToken: await mintDashboardAgentUserActorToken(watch.userId, { + environmentId: watch.environmentId, + }), + }; + + const apiClient = new ApiClient(apiOrigin, accessToken); + await apiClient.appendToSessionStream( + // Sessions are addressable by externalId, which is the chat id. + watch.chatId, + "in", + JSON.stringify({ + kind: "message", + payload: { + chatId: watch.chatId, + trigger: "action", + action: watchInvestigateAction(watch), + metadata, + }, + }) + ); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts b/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts new file mode 100644 index 00000000000..e2efc40de80 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts @@ -0,0 +1,312 @@ +/** + * The queue condition family: drain, the two depth thresholds, stall and oldest age. + * They share one freshness fence — a claim of quiet needs a reading that describes now. + */ + +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import { + formatMs, + type WatchCheckDeps, + type WatchCheckInput, + type WatchCheckOutcome, + type WatchQueueDepth, +} from "./dashboardAgentWatchCheckBase"; + +/** + * The queue-depth read both threshold kinds share. A missing queue is `terminal_unsatisfied`, + * an unreadable or stale-low depth is `unavailable`, and a stale-high one is approximate. + */ +async function readDepthOrOutcome(args: { + queue: string; + deps: WatchCheckDeps; + /** The observation to record when there is no usable reading. */ + unobserved: (verified: boolean) => WatchObservedOutcome; + /** A non-current reading at or under this is refused; one above it passes through. */ + quietLine: number; + /** + * Stateful kinds only: no non-current reading is usable, because a phantom sample would + * enter the streak as if it had been observed now. + */ + requireCurrent?: boolean; +}): Promise< + | { ok: true; depth: WatchQueueDepth; facts: Record } + | { ok: false; outcome: WatchCheckOutcome } +> { + const { queue, deps, unobserved, quietLine } = args; + const depth = await deps.readQueueDepth(queue); + + if (depth === null) { + // Only a missing queue is terminal, not an unreadable depth. + const exists = await deps.queueExists(queue); + if (!exists) { + return { + ok: false, + outcome: { + result: "terminal_unsatisfied", + facts: { queue, reason: "queue_not_found" }, + observed: unobserved(true), + }, + }; + } + return { + ok: false, + outcome: { + result: "unavailable", + facts: { queue, reason: "depth_unavailable" }, + observed: unobserved(false), + }, + }; + } + + const facts = { + queue, + depth: depth.depth, + depthSource: depth.source, + depthAsOf: depth.asOf?.toISOString() ?? null, + depthApproximate: !depth.current, + }; + + // A claim of quiet needs a reading that describes now: a stale empty bucket is never + // read as drained. + if (!depth.current && (args.requireCurrent || depth.depth <= quietLine)) { + return { + ok: false, + outcome: { + result: "unavailable", + facts: { ...facts, reason: "depth_stale" }, + observed: unobserved(false), + }, + }; + } + + return { ok: true, depth, facts }; +} + +/** + * Satisfied when the queue's current pending count is 0. The observation carries the depth + * read, so a window completing without a drain needs no second read. + */ +export async function checkBacklogDrain( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ kind: "backlog_drain", verified, depth: null }), + quietLine: 0, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth === 0 ? "satisfied" : "pending", + facts: read.facts, + observed: { kind: "backlog_drain", verified: true, depth: read.depth.depth }, + }; +} + +/** + * Satisfied when the pending count rises above `threshold`. No `terminal_unsatisfied` on a + * live queue: only the queue disappearing makes the condition impossible. + */ +export async function checkQueueDepthAbove( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_depth_above", + verified, + depth: null, + threshold: spec.threshold, + }), + quietLine: spec.threshold, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth > spec.threshold ? "satisfied" : "pending", + facts: { ...read.facts, threshold: spec.threshold }, + observed: { + kind: "queue_depth_above", + verified: true, + depth: read.depth.depth, + threshold: spec.threshold, + }, + }; +} + +/** + * The mirror of `queue_depth_above`: satisfied at or under `threshold`, which is also the quiet + * line for the freshness fence. Only the queue disappearing is terminal. + */ +export async function checkQueueDepthBelow( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_depth_below", + verified, + depth: null, + threshold: spec.threshold, + }), + quietLine: spec.threshold, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth <= spec.threshold ? "satisfied" : "pending", + facts: { ...read.facts, threshold: spec.threshold }, + observed: { + kind: "queue_depth_below", + verified: true, + depth: read.depth.depth, + threshold: spec.threshold, + }, + }; +} + +/** The stall state one check hands the next, read out of the previous facts. */ +type WatchStallState = { depth: number; notDecreasingStreak: number }; + +function readStallState( + previous: Record | null | undefined +): WatchStallState | null { + if (!previous) return null; + const depth = previous.depth; + if (typeof depth !== "number" || !Number.isFinite(depth)) return null; + const streak = previous.notDecreasingStreak; + return { + depth, + notDecreasingStreak: typeof streak === "number" && Number.isFinite(streak) ? streak : 0, + }; +} + +/** + * Satisfied when the depth fails to decrease for `ticks` consecutive checks with runs queued. + * The streak lives only in `input.previous`, a gap freezes it, and depth 0 resets it. + */ +export async function checkQueueStalled( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const previous = readStallState(input.previous); + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_stalled", + verified, + depth: null, + // Carry the streak through an unusable check. + notDecreasingStreak: previous?.notDecreasingStreak ?? 0, + ticks: spec.ticks, + }), + quietLine: 0, + requireCurrent: true, + }); + if (!read.ok) return read.outcome; + + const depth = read.depth.depth; + // A first observation has nothing to compare against, so it isn't a stalled tick. + const notDecreasingStreak = + depth === 0 || previous === null + ? 0 + : depth >= previous.depth + ? previous.notDecreasingStreak + 1 + : 0; + + const facts = { + ...read.facts, + previousDepth: previous?.depth ?? null, + notDecreasingStreak, + ticks: spec.ticks, + }; + + return { + result: depth > 0 && notDecreasingStreak >= spec.ticks ? "satisfied" : "pending", + facts, + observed: { + kind: "queue_stalled", + verified: true, + depth, + notDecreasingStreak, + ticks: spec.ticks, + }, + }; +} + +/** + * Satisfied when the oldest waiting run has waited longer than the SLA. Any stale reading is + * `unavailable` rather than compared, and an empty queue is `pending`. + */ +export async function checkQueueOldestAge( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const thresholdMs = spec.thresholdMinutes * 60_000; + const unobserved = (verified: boolean): WatchObservedOutcome => ({ + kind: "queue_oldest_age", + verified, + ageMs: null, + thresholdMinutes: spec.thresholdMinutes, + }); + + const reading = await deps.readQueueOldestAge(spec.queue); + + if (reading === null) { + const exists = await deps.queueExists(spec.queue); + if (!exists) { + return { + result: "terminal_unsatisfied", + facts: { queue: spec.queue, reason: "queue_not_found" }, + observed: unobserved(true), + }; + } + return { + result: "unavailable", + facts: { queue: spec.queue, reason: "age_unavailable" }, + observed: unobserved(false), + }; + } + + const facts = { + queue: spec.queue, + ageMs: reading.ageMs, + ageLabel: reading.ageMs === null ? null : formatMs(reading.ageMs), + ageSource: reading.source, + ageAsOf: reading.asOf?.toISOString() ?? null, + thresholdMinutes: spec.thresholdMinutes, + }; + + if (!reading.current) { + return { + result: "unavailable", + facts: { ...facts, reason: "age_stale" }, + observed: unobserved(false), + }; + } + + const observed: WatchObservedOutcome = { + kind: "queue_oldest_age", + verified: true, + ageMs: reading.ageMs, + thresholdMinutes: spec.thresholdMinutes, + }; + + return { + result: reading.ageMs !== null && reading.ageMs > thresholdMs ? "satisfied" : "pending", + facts, + observed, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts b/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts new file mode 100644 index 00000000000..ceccbb43f0f --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts @@ -0,0 +1,240 @@ +/** + * The run condition family: start, finished, failed. All three read one run row and + * label the wait with what the data actually supports. + */ + +import { + watchRunDisposition, + type WatchObservedOutcome, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { + formatMs, + type WatchCheckDeps, + type WatchCheckInput, + type WatchCheckOutcome, + type WatchRunRow, +} from "./dashboardAgentWatchCheckBase"; + +// Mirrors ~/v3/taskStatus. Kept local so this module has no server-side import. + +const FINAL_STATUSES = new Set([ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]); + +/** + * Statuses whose `queuedAt` is a leftover from the first enqueue, since resume/retry + * re-enqueues don't restamp it, so a wait computed from it isn't this attempt's. + */ +const STALE_QUEUED_AT_STATUSES = new Set(["WAITING_TO_RESUME", "RETRYING_AFTER_FAILURE", "PAUSED"]); + +export function isTerminalRunStatus(status: string): boolean { + return FINAL_STATUSES.has(status); +} + +/** Which timestamp a wait was measured from. */ +export type WatchWaitBasis = "queued_at" | "delay_until" | "created_at"; + +/** + * The wait a run has accumulated, labelled with what the data supports. A resumed, retried or + * paused run's stale `queuedAt` is never measured from. + */ +export function describeRunWait( + run: WatchRunRow, + now: Date +): { + waitMs: number | null; + waitBasis: WatchWaitBasis; + waitLabel: string; + /** True only when the wait is this attempt's queue wait. */ + queueWaitReliable: boolean; +} { + const queueWaitReliable = run.queuedAt !== null && !STALE_QUEUED_AT_STATUSES.has(run.status); + const end = run.startedAt ?? now; + + if (run.queuedAt && queueWaitReliable) { + const waitMs = Math.max(0, end.getTime() - run.queuedAt.getTime()); + return { + waitMs, + waitBasis: "queued_at", + waitLabel: `queued for ${formatMs(waitMs)}`, + queueWaitReliable, + }; + } + + if (run.delayUntil && run.delayUntil.getTime() > now.getTime()) { + return { + waitMs: null, + waitBasis: "delay_until", + waitLabel: `scheduled to start at ${run.delayUntil.toISOString()}`, + queueWaitReliable, + }; + } + + // No `queuedAt`, or one from an earlier attempt: fall back to the run's age. + const waitMs = Math.max(0, end.getTime() - run.createdAt.getTime()); + const resumeOrRetry = run.queuedAt !== null; + return { + waitMs, + waitBasis: "created_at", + waitLabel: resumeOrRetry + ? `waiting to ${run.status === "RETRYING_AFTER_FAILURE" ? "retry" : "resume"}; time from creation: ${formatMs(waitMs)}` + : `time from creation: ${formatMs(waitMs)}`, + queueWaitReliable, + }; +} + +/** + * Satisfied the moment `startedAt` exists, whatever the current status. Terminal with no + * `startedAt` can never start, so it is `terminal_unsatisfied`. + */ +export async function checkRunStart( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + // Existence was validated at creation, so the run is gone and can never start. + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_start", verified: true, status: null, started: false }, + }; + } + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + status: run.status, + queue: run.queue, + startedAt: run.startedAt?.toISOString() ?? null, + queuedAt: run.queuedAt?.toISOString() ?? null, + ...wait, + }; + const observed: WatchObservedOutcome = { + kind: "run_start", + verified: true, + status: run.status, + started: run.startedAt !== null, + }; + + if (run.startedAt) return { result: "satisfied", facts, observed }; + if (isTerminalRunStatus(run.status)) { + return { + result: "terminal_unsatisfied", + facts: { ...facts, reason: "never_started" }, + observed, + }; + } + return { result: "pending", facts, observed }; +} + +/** + * Satisfied on any terminal status. Finished and failed are both `condition_met`, so only + * `observed.finalStatus` separates them. + */ +export async function checkRunFinished( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_finished", verified: true, finalStatus: null, durationMs: null }, + }; + } + + const finished = isTerminalRunStatus(run.status); + // Execution duration only. The queue wait is reported separately. + const durationMs = + run.startedAt && run.completedAt + ? Math.max(0, run.completedAt.getTime() - run.startedAt.getTime()) + : null; + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + outcome: run.status, + startedAt: run.startedAt?.toISOString() ?? null, + completedAt: run.completedAt?.toISOString() ?? null, + durationMs, + durationLabel: durationMs === null ? null : formatMs(durationMs), + ...wait, + }; + + return { + result: finished ? "satisfied" : "pending", + facts, + observed: { + kind: "run_finished", + verified: true, + // Only a terminal status is a final status. + finalStatus: finished ? run.status : null, + durationMs, + }, + }; +} + +/** + * A failing terminal status satisfies this; a successful completion or a cancellation makes + * the condition impossible rather than merely unmet. + */ +export async function checkRunFailed( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_failed", verified: true, finalStatus: null, durationMs: null }, + }; + } + + const finished = isTerminalRunStatus(run.status); + const durationMs = + run.startedAt && run.completedAt + ? Math.max(0, run.completedAt.getTime() - run.startedAt.getTime()) + : null; + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + outcome: run.status, + startedAt: run.startedAt?.toISOString() ?? null, + completedAt: run.completedAt?.toISOString() ?? null, + durationMs, + durationLabel: durationMs === null ? null : formatMs(durationMs), + ...wait, + }; + const observed: WatchObservedOutcome = { + kind: "run_failed", + verified: true, + finalStatus: finished ? run.status : null, + durationMs, + }; + + if (!finished) return { result: "pending", facts, observed }; + + return { + result: watchRunDisposition(run.status) === "failed" ? "satisfied" : "terminal_unsatisfied", + facts: { + ...facts, + ...(watchRunDisposition(run.status) === "failed" ? {} : { reason: "cannot_fail_now" }), + }, + observed, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts b/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts new file mode 100644 index 00000000000..0bd35deb48d --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts @@ -0,0 +1,516 @@ +/** + * The watch backstop for a dead tick chain. Finalization runs even with no agent project, or rows + * would stay `active` forever; what can't be handed over stays owed. Guarded, so a re-run no-ops. + */ + +import { + cancelWatch, + deleteTerminalWatchesOlderThan, + deleteWatchSubmissionsOlderThan, + listExpiredActiveWatches, + listWatchBatchGroupsToArm, + listWatchesAwaitingDelivery, + transitionWatchCondition, + type Watch, + type WatchBatchGroup, +} from "@internal/dashboard-agent-db"; +import { + watchResolutionForCheck, + watchResolutionToWireStatus, + type WatchObservedOutcome, + type WatchResolution, +} from "@internal/dashboard-agent-contracts"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { enqueueWatchFiredAlert } from "~/services/dashboardAgentWatchAlerts.server"; +import { + checkWatch, + previousCheckFacts, + type WatchCheckDeps, + type WatchCheckOutcome, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { mapWithConcurrency, shareReads } from "~/services/dashboardAgentWatchBatch.server"; +import { isDashboardAgentConfigured } from "~/services/dashboardAgent.server"; +import { + armDashboardAgentWatchBatch, + authorizeWatchEnvironment, + scheduleWatchDelivery, + type WatchAuthorization, +} from "~/services/dashboardAgentWatches.server"; +import { logger } from "~/services/logger.server"; + +/** + * How long past `expiresAt` a watch is left to the tick chain. Only has to cover a late + * tick: the chain's own final check happens within a cadence of the deadline. + */ +export const WATCH_EXPIRY_GRACE_MS = 2 * 60 * 1000; + +/** + * How long a resolved watch may owe its wake before the sweep recovers it. Long enough that + * the recovery can't race a delivery still in flight. + */ +export const WATCH_DELIVERY_GRACE_MS = 5 * 60 * 1000; + +/** Per-run cap for each half of the sweep. Oldest first, so the rest land next run. */ +const SWEEP_BATCH_LIMIT = 100; + +/** How long a terminal watch is kept. Its outcome also lives in the chat transcript. */ +export const WATCH_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; + +/** Higher than the other caps: retention is one statement, not a row-at-a-time loop. */ +const RETENTION_BATCH_LIMIT = 500; + +/** + * How many rows one sweep handles at once. An incident expires a whole group together, and a + * bound is what stops one slow tenant spending the entire visibility window. + */ +const SWEEP_CONCURRENCY = 8; + +/** What one finalization did. */ +export type WatchFinalizeOutcome = + | "fired" + | "expired" + /** The user lost access: cancelled, and deliberately not narrated. */ + | "cancelled" + /** A tick (or another sweep) resolved it first. */ + | "already_resolved"; + +export type WatchSweepResult = { + /** Overdue active rows seen. */ + overdue: number; + fired: number; + expired: number; + cancelled: number; + alreadyResolved: number; + /** Resolved rows whose wake was still owed. */ + undelivered: number; + /** Wakes handed back to the watcher task. */ + redelivered: number; + /** Decided but not handed over, with no agent project. They stay owed. */ + deliveryDeferred: number; + /** Long-terminal rows dropped by retention. */ + purged: number; + /** Ledger rows dropped by retention. */ + purgedSubmissions: number; + failed: number; +}; + +export type WatchSweepDeps = { + now?: () => Date; + limit?: number; + /** Overdue `active` rows. */ + listOverdue?: (params: { now: Date; limit: number }) => Promise; + /** Resolved rows whose wake is still owed. */ + listAwaitingDelivery?: (params: { olderThan: Date; limit: number }) => Promise; + /** Re-authorization of the watch's initiating user. */ + authorize?: (watch: Watch) => Promise; + /** The environment readers the final check runs against. */ + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + /** Hand the wake back to the watcher task. Must throw if it can't be scheduled. */ + deliver?: (watch: Watch) => Promise; + /** Gates the delivery half only. Finalization never depends on it. */ + configured?: () => boolean; + /** Drop terminal rows older than `before`. Returns how many went. */ + purgeTerminal?: (params: { before: Date; limit: number }) => Promise; + /** Drop submission-ledger rows older than `before`. */ + purgeSubmissions?: (params: { before: Date; limit: number }) => Promise; + /** How many rows are handled at once. */ + concurrency?: number; +}; + +/** + * One re-authorization per (user, org, project, environment) for the whole sweep. An incident + * expires a group together, and every row of it names the same access question. + */ +function authorizeOncePerSweep( + deps: WatchSweepDeps +): (watch: Watch) => Promise { + const authorize = deps.authorize ?? defaultAuthorize; + const seen = new Map>(); + return (watch) => { + const key = `${watch.userId}:${watch.organizationId}:${watch.projectId}:${watch.environmentId}`; + const cached = seen.get(key); + if (cached) return cached; + const pending = authorize(watch); + seen.set(key, pending); + return pending; + }; +} + +/** + * One set of readers per environment, read-shared the way the batch's are: `now` is fixed for + * the sweep, so the same read can't answer two ways. + */ +function readersOncePerSweep( + deps: WatchSweepDeps +): (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps { + const build = deps.checkDeps ?? watchCheckDeps; + const seen = new Map(); + return (environment, now) => { + const cached = seen.get(environment.id); + if (cached) return cached; + const readers = shareReads(build(environment, now)); + seen.set(environment.id, readers); + return readers; + }; +} + +/** + * Facts for a swept expiry, in the same shape the tick writes. `verified: false` means the + * condition couldn't be evaluated, and carries the last observation instead. + */ +function expiredFacts( + watch: Watch, + args: { verified: boolean; reason: string; facts?: Record } +): Record { + return { + verified: args.verified, + reason: args.reason, + expiredAt: watch.expiresAt.toISOString(), + checks: watch.tickCount, + ...(args.verified + ? (args.facts ?? {}) + : { + lastObservedAt: watch.lastCheckedAt?.toISOString(), + lastObservation: watch.lastResult, + }), + }; +} + +/** + * How a final check's verdict resolves the row. The final read is a real evaluation, so only + * `pending` and `unavailable` become `window_completed`. `watchResolutionForCheck` owns that. + */ +function resolutionFor( + watch: Watch, + outcome: WatchCheckOutcome +): { + resolution: WatchResolution; + observed: WatchObservedOutcome; + facts: Record; +} { + // Always non-null here: this is the boundary evaluation. + const resolution = watchResolutionForCheck(outcome.result, true)!; + + switch (outcome.result) { + case "satisfied": + return { + resolution, + observed: outcome.observed, + facts: { verified: true, ...outcome.facts }, + }; + case "terminal_unsatisfied": + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { + verified: true, + reason: "terminal_unsatisfied", + facts: outcome.facts, + }), + }; + case "pending": + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { + verified: true, + reason: "not_met_by_expiry", + facts: outcome.facts, + }), + }; + default: + // The check couldn't run, so the window completes unverified. + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { verified: false, reason: "unverified_at_expiry" }), + }; + } +} + +/** + * Finalize one overdue watch. Re-authorization comes first, before the final check reads + * anything; `canDeliver: false` stops at the resolution, leaving the wake owed. + */ +export async function finalizeOverdueWatch( + watch: Watch, + deps: WatchSweepDeps & { canDeliver?: boolean } = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const authorize = deps.authorize ?? defaultAuthorize; + const buildCheckDeps = deps.checkDeps ?? watchCheckDeps; + const deliver = deps.deliver ?? scheduleWatchDelivery; + const canDeliver = deps.canDeliver ?? true; + + const authorization = await authorize(watch); + if (!authorization.ok) { + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "access_revoked" }); + logger.info("Dashboard agent watch sweep: cancelled a watch whose access was revoked", { + watchId: watch.id, + }); + return "cancelled"; + } + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const outcome = await checkWatch( + watch.spec, + buildCheckDeps(authorization.environment, now), + // A stateful condition is a transition across checks, so the boundary evaluation needs + // the previous facts to see one — and to record what it saw, not a reset. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch sweep: the final check failed", { + watchId: watch.id, + error, + }) + ); + + const resolved = resolutionFor(watch, outcome); + const transitioned = await transitionWatchCondition(dashboardAgentDb, { + id: watch.id, + resolution: resolved.resolution, + observedOutcome: resolved.observed, + lastResult: resolved.facts, + }); + + // Guarded on `active`: a tick that resolved it first keeps its outcome and delivery. + if (!transitioned) return "already_resolved"; + + if (resolved.resolution === "condition_met") { + // Keyed on the watch, so the wake's own notification can't double-alert it. + try { + await enqueueWatchFiredAlert(transitioned, "fired"); + } catch (error) { + logger.error("Dashboard agent watch sweep: failed to enqueue the fired alert", { + watchId: watch.id, + error, + }); + } + } + + // Throws if it can't be scheduled, leaving the row terminal with its delivery owed. + if (canDeliver) await deliver(transitioned); + return watchResolutionToWireStatus(resolved.resolution); +} + +/** + * Recover one owed wake, unconditionally: this sweep can't tell whether the user was already + * told. Whether the wake needs prose is decided where the transcript can be read. + */ +export async function recoverWatchDelivery(watch: Watch, deps: WatchSweepDeps = {}): Promise { + const deliver = deps.deliver ?? scheduleWatchDelivery; + await deliver(watch); +} + +/** + * One sweep: finalize what is overdue, then recover what was never delivered. Each row is + * handled on its own, and the run throws at the end if any failed so the job is retried. + */ +export async function sweepDashboardAgentWatches( + deps: WatchSweepDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const limit = deps.limit ?? SWEEP_BATCH_LIMIT; + const configured = deps.configured ?? isDashboardAgentConfigured; + const listOverdue = + deps.listOverdue ?? ((params) => listExpiredActiveWatches(dashboardAgentDb, params)); + const listAwaitingDelivery = + deps.listAwaitingDelivery ?? + ((params) => listWatchesAwaitingDelivery(dashboardAgentDb, params)); + const purgeTerminal = + deps.purgeTerminal ?? ((params) => deleteTerminalWatchesOlderThan(dashboardAgentDb, params)); + const purgeSubmissions = + deps.purgeSubmissions ?? + ((params) => deleteWatchSubmissionsOlderThan(dashboardAgentDb, params)); + + const result: WatchSweepResult = { + overdue: 0, + fired: 0, + expired: 0, + cancelled: 0, + alreadyResolved: 0, + undelivered: 0, + redelivered: 0, + deliveryDeferred: 0, + purged: 0, + purgedSubmissions: 0, + failed: 0, + }; + + // Gates the hand-off only: the rows still have to be finalized. + const canDeliver = configured(); + if (!canDeliver) { + logger.warn( + "Dashboard agent watch sweep: the agent isn't configured, so wakes can't be delivered — finalizing only" + ); + } + + const overdue = await listOverdue({ + now: new Date(now.getTime() - WATCH_EXPIRY_GRACE_MS), + limit, + }); + result.overdue = overdue.length; + + // The authorization and the readers are resolved once for the whole sweep, so a group of + // expiries costs one of each rather than one per row. + const perSweep: WatchSweepDeps & { canDeliver: boolean } = { + ...deps, + now: () => now, + canDeliver, + authorize: authorizeOncePerSweep(deps), + checkDeps: readersOncePerSweep(deps), + }; + + const finalized = await mapWithConcurrency( + overdue, + deps.concurrency ?? SWEEP_CONCURRENCY, + async (watch) => { + try { + return await finalizeOverdueWatch(watch, perSweep); + } catch (error) { + logger.error("Dashboard agent watch sweep: failed to finalize a watch", { + watchId: watch.id, + error, + }); + return null; + } + } + ); + + for (const outcome of finalized) { + if (outcome === null) result.failed++; + else if (outcome === "fired") result.fired++; + else if (outcome === "expired") result.expired++; + else if (outcome === "cancelled") result.cancelled++; + else result.alreadyResolved++; + // Resolved, but nothing carried the wake away: it stays owed. + if (!canDeliver && (outcome === "fired" || outcome === "expired")) { + result.deliveryDeferred++; + } + } + + // Skipped without an agent project: the rows keep their owed wake for the next sweep. + if (canDeliver) { + const owed = await listAwaitingDelivery({ + olderThan: new Date(now.getTime() - WATCH_DELIVERY_GRACE_MS), + limit, + }); + result.undelivered = owed.length; + + const recovered = await mapWithConcurrency( + owed, + deps.concurrency ?? SWEEP_CONCURRENCY, + async (watch) => { + try { + await recoverWatchDelivery(watch, deps); + return true; + } catch (error) { + logger.error("Dashboard agent watch sweep: failed to recover a wake", { + watchId: watch.id, + error, + }); + return false; + } + } + ); + + for (const ok of recovered) { + if (ok) result.redelivered++; + else result.failed++; + } + } + + // Retention runs last, over rows both halves are finished with. Its own try/catch so a + // lost retention pass can't mask the other failures. + try { + const before = new Date(now.getTime() - WATCH_RETENTION_MS); + result.purged = await purgeTerminal({ before, limit: RETENTION_BATCH_LIMIT }); + // The ledger's rows age out on the same window: past it no client is still retrying. + result.purgedSubmissions = await purgeSubmissions({ before, limit: RETENTION_BATCH_LIMIT }); + } catch (error) { + result.failed++; + logger.error("Dashboard agent watch sweep: failed to purge terminal watches", { error }); + } + + if (result.failed > 0) { + throw new Error(`The dashboard agent watch sweep failed on ${result.failed} watches`); + } + + return result; +} + +export type WatchBatchRearmResult = { + /** Groups with active watches and no live chain. */ + stale: number; + armed: number; + failed: number; +}; + +export type WatchBatchRearmDeps = { + now?: () => Date; + limit?: number; + /** Groups whose chain is missing, stopped, or has gone silent. */ + listGroups?: (params: { now: Date; limit: number }) => Promise; + /** Start a chain for one group. */ + arm?: (params: { + environmentId: string; + cadenceMinutes: number; + now?: Date; + }) => Promise<{ running: boolean }>; + configured?: () => boolean; +}; + +const REARM_BATCH_LIMIT = 200; + +/** + * Re-arm batch chains that died. `armWatchBatch` re-checks the same timestamp in the statement + * that arms, so racing a merely slow chain arms nothing and two runs start one chain. + */ +export async function rearmDashboardAgentWatchBatches( + deps: WatchBatchRearmDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const limit = deps.limit ?? REARM_BATCH_LIMIT; + const configured = deps.configured ?? isDashboardAgentConfigured; + const listGroups = + deps.listGroups ?? ((params) => listWatchBatchGroupsToArm(dashboardAgentDb, params)); + const arm = deps.arm ?? armDashboardAgentWatchBatch; + + const result: WatchBatchRearmResult = { stale: 0, armed: 0, failed: 0 }; + + // Nothing to trigger a chain into. The expiry half still finalizes the rows. + if (!configured()) return result; + + const groups = await listGroups({ now, limit }); + result.stale = groups.length; + + for (const group of groups) { + try { + const { running } = await arm({ ...group, now }); + if (running) result.armed++; + } catch (error) { + result.failed++; + logger.error("Dashboard agent watch sweep: failed to re-arm a batch chain", { + ...group, + error, + }); + } + } + + if (result.failed > 0) { + throw new Error(`The dashboard agent batch re-arm failed on ${result.failed} groups`); + } + + return result; +} + +function defaultAuthorize(watch: Watch): Promise { + return authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchToken.server.ts b/apps/webapp/app/services/dashboardAgentWatchToken.server.ts new file mode 100644 index 00000000000..dc56691f560 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchToken.server.ts @@ -0,0 +1,181 @@ +import { generateJWT, validateJWT } from "@trigger.dev/core/v3/jwt"; +import { env } from "~/env.server"; + +/** + * The credential the watcher task presents to the private check endpoint. It names one watch, is + * kept apart from the UAT by a disjoint prefix and `kind`, and is re-minted, never persisted. + */ + +export const WATCH_TOKEN_PREFIX = "tr_daw_"; + +/** Distinguishes a watch token from every other SESSION_SECRET-signed JWT. */ +const WATCH_TOKEN_KIND = "dashboard_agent_watch"; + +/** Mirrors the UAT's `act.client` with a value no UAT uses. Verified, so no replay. */ +const WATCH_TOKEN_CLIENT = "dashboard-agent-watch"; + +/** + * How long past `expiresAt` the token stays valid. The final check happens after the + * deadline, so the token has to outlive the watch by enough to cover a late tick. + */ +export const WATCH_TOKEN_GRACE_MS = 60 * 60 * 1000; + +export type WatchTokenClaims = { + watchId: string; + /** Token expiry (seconds since epoch), i.e. `expiresAt` + grace. */ + expiresAtSeconds: number; +}; + +export function isDashboardAgentWatchToken(token: string): boolean { + return token.startsWith(WATCH_TOKEN_PREFIX); +} + +/** Deterministic: the same inputs produce the same string. */ +export async function signDashboardAgentWatchToken( + secret: string, + opts: { watchId: string; expiresAt: Date; graceMs?: number } +): Promise { + const expirationTime = Math.floor( + (opts.expiresAt.getTime() + (opts.graceMs ?? WATCH_TOKEN_GRACE_MS)) / 1000 + ); + + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: WATCH_TOKEN_KIND, + // `sub` is the watch: a watch token never authenticates a user. + sub: opts.watchId, + act: { client: WATCH_TOKEN_CLIENT }, + }, + expirationTime, + omitIssuedAt: true, + }); + + return `${WATCH_TOKEN_PREFIX}${jwt}`; +} + +/** `undefined` for anything but a valid watch token, including a valid user-actor token. */ +export async function verifyDashboardAgentWatchToken( + secret: string, + token: string +): Promise { + if (!isDashboardAgentWatchToken(token)) return; + + const result = await validateJWT(token.slice(WATCH_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== WATCH_TOKEN_KIND) return; + if (typeof payload.sub !== "string" || payload.sub.length === 0) return; + + const act = payload.act as { client?: string } | undefined; + if (act?.client !== WATCH_TOKEN_CLIENT) return; + if (typeof payload.exp !== "number") return; + + return { watchId: payload.sub, expiresAtSeconds: payload.exp }; +} + +export function mintDashboardAgentWatchToken(opts: { + watchId: string; + expiresAt: Date; +}): Promise { + return signDashboardAgentWatchToken(env.SESSION_SECRET, opts); +} + +export function verifyWatchTokenFromRequest(token: string): Promise { + return verifyDashboardAgentWatchToken(env.SESSION_SECRET, token); +} + +/** + * Chain tokens name a whole (environment, cadence) group. The prefix is disjoint from + * `tr_daw_` rather than nested under it, so neither verifier sees the other's tokens. + */ +export const WATCH_BATCH_TOKEN_PREFIX = "tr_dab_"; + +const WATCH_BATCH_TOKEN_KIND = "dashboard_agent_watch_batch"; +const WATCH_BATCH_TOKEN_CLIENT = "dashboard-agent-watch-batch"; + +/** + * How long a chain's token lives. A chain has no deadline to pin it to, but it must still + * expire; an expired one is self-healing via the re-arm backstop. + */ +export const WATCH_BATCH_TOKEN_TTL_MS = 30 * 24 * 60 * 60 * 1000; + +export type WatchBatchTokenClaims = { environmentId: string; cadenceMinutes: number }; + +/** + * Like a watch token, a chain token names one group and carries no authority of its own: the + * batch check re-authorizes every watch's initiating user against that watch's snapshot. + */ +export async function signDashboardAgentWatchBatchToken( + secret: string, + opts: { environmentId: string; cadenceMinutes: number; expiresAt: Date } +): Promise { + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: WATCH_BATCH_TOKEN_KIND, + // The group, not a user and not a watch. + sub: `${opts.environmentId}:${opts.cadenceMinutes}`, + act: { client: WATCH_BATCH_TOKEN_CLIENT }, + }, + expirationTime: Math.floor(opts.expiresAt.getTime() / 1000), + omitIssuedAt: true, + }); + + return `${WATCH_BATCH_TOKEN_PREFIX}${jwt}`; +} + +/** `undefined` for anything that isn't a valid, unexpired chain token. */ +export async function verifyDashboardAgentWatchBatchToken( + secret: string, + token: string +): Promise { + if (!token.startsWith(WATCH_BATCH_TOKEN_PREFIX)) return; + + const result = await validateJWT(token.slice(WATCH_BATCH_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== WATCH_BATCH_TOKEN_KIND) return; + const act = payload.act as { client?: string } | undefined; + if (act?.client !== WATCH_BATCH_TOKEN_CLIENT) return; + if (typeof payload.sub !== "string") return; + + // The cadence is the last segment, split from the right so a colon in an environment id + // could never confuse it. + const separator = payload.sub.lastIndexOf(":"); + if (separator <= 0) return; + const environmentId = payload.sub.slice(0, separator); + const cadenceMinutes = Number(payload.sub.slice(separator + 1)); + if (!Number.isInteger(cadenceMinutes) || cadenceMinutes <= 0) return; + + return { environmentId, cadenceMinutes }; +} + +export function mintDashboardAgentWatchBatchToken(opts: { + environmentId: string; + cadenceMinutes: number; + now?: Date; +}): Promise { + const now = opts.now ?? new Date(); + return signDashboardAgentWatchBatchToken(env.SESSION_SECRET, { + environmentId: opts.environmentId, + cadenceMinutes: opts.cadenceMinutes, + expiresAt: new Date(now.getTime() + WATCH_BATCH_TOKEN_TTL_MS), + }); +} + +export function verifyWatchBatchTokenFromRequest( + token: string +): Promise { + return verifyDashboardAgentWatchBatchToken(env.SESSION_SECRET, token); +} + +/** The bearer value from an `Authorization: Bearer …` header, if present. */ +export function bearerToken(request: Request): string | undefined { + const raw = request.headers.get("Authorization"); + if (!raw) return undefined; + const value = raw.replace(/^Bearer /, "").trim(); + return value.length > 0 ? value : undefined; +} diff --git a/apps/webapp/app/services/dashboardAgentWatches.server.ts b/apps/webapp/app/services/dashboardAgentWatches.server.ts new file mode 100644 index 00000000000..f6e4a33924d --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatches.server.ts @@ -0,0 +1,1165 @@ +/** + * Watches, webapp half: creation, the re-authorization a background check passes, and the + * chat-delete cascade. The row's snapshot is immutable, and no target comes from client input. + */ + +import { + MAX_ACTIVE_WATCHES_PER_CHAT, + appendChatMessageOnce, + armWatchBatch, + cancelWatch, + chatExists, + claimWatchSubmission, + createChat, + createWatch, + generateWatchId, + getChatWatchContext, + getWatch, + getWatchSubmission, + listActiveWatchesForChats as listActiveWatchesForChatsQuery, + precheckWatchCreation, + recordWatchSubmissionOutcome, + reopenWatchSubmission, + softDeleteChat, + stopWatchBatch, + type ChatWatchContext, + type PersistedWatchSpec, + type Watch, + type WatchStatus, + type WatchSubmission, +} from "@internal/dashboard-agent-db"; +import { + VIEW_BLOCK_VERSION, + WATCH_CONFIRMATION_MESSAGE_ID_PREFIX, + WATCH_REQUEST_MESSAGE_ID_PREFIX, + watchConfirmationBlockBody, + watchDraftSchema, + watchIdentity, + watchOneShotBlockBody, + watchRequestSentence, + watchSubjectLabel, + type WatchDraft, + type WatchExternalNotification, + type WatchObservedOutcome, + type WatchResolution, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { createHash } from "node:crypto"; +import { TriggerClient } from "@trigger.dev/sdk"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { authIncludeWithParent, toAuthenticated } from "~/models/runtimeEnvironment.server"; +import { isReportKey } from "~/presenters/v3/reports/report-registry"; +import { + dashboardAgentApiOrigin, + isDashboardAgentConfigured as isDashboardAgentConfiguredDefault, +} from "~/services/dashboardAgent.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { + checkWatch, + type WatchCheckDeps, + type WatchCheckOutcome, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCreationCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { subscribeUserToWatchAlerts } from "~/services/dashboardAgentWatchAlerts.server"; +import { + mintDashboardAgentWatchBatchToken, + mintDashboardAgentWatchToken, +} from "~/services/dashboardAgentWatchToken.server"; +import { canAccessDashboardAgent } from "~/v3/canAccessDashboardAgent.server"; + +/** The task that polls a watch. Lives in the agent project, triggered by us. */ +export const WATCH_TASK_ID = "dashboard-agent-watch"; + +export { MAX_ACTIVE_WATCHES_PER_CHAT }; + +export type WatchAuthorization = + | { ok: true; environment: AuthenticatedEnvironment } + | { ok: false; reason: "access_revoked" }; + +/** + * Re-authorize a watch's initiating user against the row's immutable project/environment; a + * partial pass is `access_revoked`. The membership-scoped query is the tenant floor here. + */ +export async function authorizeWatchEnvironment(params: { + userId: string; + organizationId: string; + projectId: string; + environmentId: string; +}): Promise { + // The primary, not the replica: replica lag would extend access the user has lost. + const environment = await prisma.runtimeEnvironment.findFirst({ + where: { + id: params.environmentId, + // The watch's snapshot has to still describe this environment. + projectId: params.projectId, + organizationId: params.organizationId, + archivedAt: null, + project: { deletedAt: null }, + organization: { deletedAt: null, members: { some: { userId: params.userId } } }, + OR: [ + { type: { in: ["PREVIEW", "STAGING", "PRODUCTION"] } }, + // Dev environments are per-member: only their owner may read them. + { type: "DEVELOPMENT", orgMember: { userId: params.userId } }, + ], + }, + include: authIncludeWithParent, + }); + + if (!environment) return { ok: false, reason: "access_revoked" }; + + // The gate only reads `isAdmin` while the admin preview is on, so this read is skipped + // otherwise: it runs on every watch check, batch authorization and sweep finalisation. + let isAdmin = false; + if (env.DASHBOARD_AGENT_ADMIN_PREVIEW === "1") { + // Primary for the same reason as the membership read above. + const user = await prisma.user.findFirst({ + where: { id: params.userId }, + select: { admin: true }, + }); + if (!user) return { ok: false, reason: "access_revoked" }; + isAdmin = user.admin; + } + + const allowed = await canAccessDashboardAgent({ + userId: params.userId, + isAdmin, + // A background check is never an impersonated session. + isImpersonating: false, + organizationSlug: environment.organization.slug, + orgFeatureFlags: environment.organization.featureFlags as Record | null, + }); + if (!allowed) return { ok: false, reason: "access_revoked" }; + + return { ok: true, environment: toAuthenticated(environment) }; +} + +/** + * The same authorization by environment id alone, for the creation path with no watch row + * yet. The id lookup is unscoped and proves nothing; `authorizeWatchEnvironment` is the gate. + */ +export async function authorizeWatchEnvironmentById(params: { + userId: string; + environmentId: string; +}): Promise { + const environment = await $replica.runtimeEnvironment.findFirst({ + where: { id: params.environmentId }, + select: { organizationId: true, projectId: true }, + }); + if (!environment) return null; + + const authorization = await authorizeWatchEnvironment({ + userId: params.userId, + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: params.environmentId, + }); + return authorization.ok ? authorization.environment : null; +} + +export type CreateWatchErrorCode = + | "limit_reached" + | "duplicate" + | "invalid_target" + | "chat_not_found" + | "not_configured" + | "internal"; + +/** + * Either a watch is now running (`watching: true`), or the immediate check answered and no + * row exists at all (`watching: false`), which never enters the delivery state machine. + */ +export type CreateDashboardAgentWatchResult = + | { + ok: true; + watching: true; + watchId: string; + identity: string; + status: WatchStatus; + expiresAt: Date; + /** Set when the creation-time check couldn't run. The watch is active anyway. */ + unavailable?: boolean; + } + | { + ok: true; + watching: false; + identity: string; + /** `satisfied` (already true) or `terminal_unsatisfied` (can't happen now). */ + immediate: WatchCheckOutcome; + } + | { + ok: false; + error: string; + code: CreateWatchErrorCode; + /** The watch already covering this condition, on `duplicate`. */ + existingId?: string | null; + }; + +/** + * Existence check for the thing a spec points at, in this environment. `error_recurrence` + * has nothing to validate: zero occurrences so far is the normal case. + */ +async function validateWatchTarget(spec: WatchSpec, deps: WatchCheckDeps): Promise { + switch (spec.kind) { + case "run_start": + case "run_finished": + case "run_failed": + return (await deps.readRun(spec.runId)) !== null; + case "backlog_drain": + case "queue_depth_above": + case "queue_depth_below": + case "queue_stalled": + case "queue_oldest_age": + return await deps.queueExists(spec.queue); + case "error_recurrence": + return spec.fingerprint.length > 0; + case "health_recovery": + return isReportKey(spec.report); + } +} + +/** + * Create a watch for an already-authorized context. The order is load-bearing (cap, dedup, + * immediate check, create), and a first tick that can't be scheduled cancels the row. + */ +export async function createDashboardAgentWatch(params: { + environment: AuthenticatedEnvironment; + userId: string; + chatId: string; + spec: WatchSpec; + /** Consent to investigate after an attention outcome. Never inferred. */ + investigateOnAttention?: boolean; + /** Reserved by the submission ledger, so a converging retry finds the row by id. */ + watchId?: string; + now?: Date; + /** IO seams: tests inject fakes here instead of mocking the readers. */ + deps?: { + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + scheduleTick?: typeof scheduleWatchTick; + /** Skip the real trigger-config gate when a tick scheduler is injected. */ + configured?: () => boolean; + }; +}): Promise { + const { environment, userId, chatId, spec } = params; + const now = params.now ?? new Date(); + // Creation reads the target on the primary; the polling checks stay on the replica. + const buildCheckDeps = params.deps?.checkDeps ?? watchCreationCheckDeps; + const scheduleTick = params.deps?.scheduleTick ?? scheduleWatchTick; + const isDashboardAgentConfigured = params.deps?.configured ?? isDashboardAgentConfiguredDefault; + const checkDeps = buildCheckDeps(environment, now); + + if (!isDashboardAgentConfigured()) { + return { + ok: false, + code: "not_configured", + error: "The dashboard agent is not configured, so watches can't be scheduled.", + }; + } + + if (!(await validateWatchTarget(spec, checkDeps))) { + return { + ok: false, + code: "invalid_target", + error: "That target doesn't exist in this environment.", + }; + } + + const identity = watchIdentity(spec); + + // Advisory only: `createWatch` below re-applies both guardrails atomically and + // stays the authority. + const precheck = await precheckWatchCreation(dashboardAgentDb, { + chatId, + projectId: environment.projectId, + environmentId: environment.id, + identity, + }); + if (!precheck.ok) return creationGuardrailError(precheck); + + // `since` is server-set so the model can't backdate a recurrence window. + const persistedSpec: PersistedWatchSpec = + spec.kind === "error_recurrence" ? { ...spec, since: now.toISOString() } : spec; + + // Answer in the same turn when the condition has already happened. + const immediate = await checkWatch(persistedSpec, checkDeps, { now, since: now }, (error) => + logger.error("Dashboard agent watch: immediate check failed", { chatId, identity, error }) + ); + + if (immediate.result === "satisfied" || immediate.result === "terminal_unsatisfied") { + // Nothing is persisted: no row means no delivery claim and no wake. + return { ok: true, watching: false, identity, immediate }; + } + + const expiresAt = new Date(now.getTime() + spec.maxHours * 60 * 60 * 1000); + + const created = await createWatch(dashboardAgentDb, { + ...(params.watchId ? { id: params.watchId } : {}), + chatId, + identity, + spec: persistedSpec, + organizationId: environment.organizationId, + projectId: environment.projectId, + // The external ref travels with the row: the agent can't translate the internal id. + projectRef: environment.project.externalRef, + environmentId: environment.id, + userId, + expiresAt, + investigateOnAttention: params.investigateOnAttention === true, + }); + + if (!created.ok) { + if (created.error === "chat_not_found") { + // The chat was deleted mid-create. The query layer re-reads it under the + // per-chat lock, so nothing was written. + return { + ok: false, + code: "chat_not_found", + error: "That chat no longer exists, so nothing is being watched.", + }; + } + return creationGuardrailError(created); + } + + const watch = created.watch; + const token = await mintDashboardAgentWatchToken({ watchId: watch.id, expiresAt }); + + try { + await scheduleTick({ + watchId: watch.id, + token, + delayMinutes: spec.checkEveryMinutes, + // Each invocation claims its own generation atomically, so the first is + // `tickCount + 1`. + tick: watch.tickCount + 1, + }); + } catch (error) { + logger.error("Dashboard agent watch: failed to schedule the first tick", { + id: watch.id, + error, + }); + // Cancelled rather than resolved, because the condition was never evaluated. + // Cancellation is silent, so no wake is sent. + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "scheduling_failed" }); + return { + ok: false, + code: "internal", + error: "The watch couldn't be scheduled. Nothing is being watched.", + }; + } + + return { + ok: true, + watching: true, + watchId: watch.id, + identity, + status: "active", + expiresAt, + ...(immediate.result === "unavailable" ? { unavailable: true } : {}), + }; +} + +/* ------------------------------------------------------------------ * + * The card submit: a durable record of the request, then the watch + * ------------------------------------------------------------------ */ + +/** A stored transcript record. Deterministic, so a retry rewrites the same bytes. */ +export type WatchTranscriptMessage = { + id: string; + role: "user" | "assistant"; + parts: unknown[]; +}; + +/** A submit can also refuse a request id that arrives carrying a different draft. */ +export type SubmitWatchErrorCode = CreateWatchErrorCode | "request_conflict"; + +export type SubmitWatchCardResult = + | { + ok: true; + chatId: string; + watching: boolean; + watchId: string | null; + /** The request record and the confirmation, in transcript order. */ + messages: WatchTranscriptMessage[]; + /** Nothing was created: this call replayed a recorded outcome. */ + repaired: boolean; + } + | { + ok: false; + code: SubmitWatchErrorCode; + error: string; + existingId?: string | null; + /** Set once a chat exists, so the caller can still open it. */ + chatId?: string; + }; + +/** + * A fresh panel's chat id, derived from the request id so a retried submit lands in the + * chat the first attempt created instead of leaving an empty one behind. + * + * The whole tenancy of the request is mixed in, not just the user: `clientRequestId` is + * client-chosen, and one user can be in several organizations, so a user id alone lets two + * organizations derive the same id — where `createChat(...).onConflictDoNothing()` keeps + * the first org's chat and the second org's records land in it. + */ +function chatIdForRequest(params: { + organizationId: string; + userId: string; + environmentId: string; + clientRequestId: string; +}): string { + const digest = createHash("sha256") + .update( + `${params.organizationId}:${params.userId}:${params.environmentId}:${params.clientRequestId}` + ) + .digest("hex"); + return `chat_${digest.slice(0, 24)}`; +} + +/** + * A spec's comparable form. `since` is server-set on every attempt, so it is excluded: + * two attempts at the same request differ by it and are still the same request. + */ +function comparableSpec(spec: WatchSpec | PersistedWatchSpec): string { + const entries = Object.entries(spec as Record) + .filter(([key]) => key !== "since") + .sort(([a], [b]) => a.localeCompare(b)); + return JSON.stringify(entries); +} + +/** + * Whether an existing watch is the one this submission asked for. A retry is byte-identical, + * so anything else — a different window, cadence, note or investigate consent — is a genuinely + * different request and still conflicts. `notifyExternally` is not on the watch row, so it is + * compared through the ledger's draft digest instead, which covers the whole configuration. + */ +function isSameWatchRequest(existing: Watch, draft: WatchDraft): boolean { + return ( + comparableSpec(existing.spec) === comparableSpec(draft.spec) && + existing.investigateOnAttention === draft.followUp.investigateOnAttention + ); +} + +/** The record of what the user confirmed. Written with no model call. */ +function requestMessage(clientRequestId: string, draft: WatchDraft): WatchTranscriptMessage { + return { + id: `${WATCH_REQUEST_MESSAGE_ID_PREFIX}${clientRequestId}`, + role: "user", + parts: [ + { type: "text", text: watchRequestSentence({ spec: draft.spec, followUp: draft.followUp }) }, + ], + }; +} + +/** The confirmation block, keyed on the watch so a repair rebuilds exactly the same record. */ +function confirmationMessage(args: { + id: string; + blockId: string; + body: Record; +}): WatchTranscriptMessage { + return { + id: args.id, + role: "assistant", + parts: [ + { + type: "data-view", + data: { + blocks: [ + { ...args.body, revision: 0, version: VIEW_BLOCK_VERSION, id: `watch:${args.blockId}` }, + ], + }, + }, + ], + }; +} + +/** + * A submitted draft's comparable digest: the whole confirmed configuration, `notifyExternally` + * included. It is user consent, and the transcript records it, so a retry that flips it is a + * different request — not something to converge on behind the durable record. + */ +function draftDigest(draft: WatchDraft): string { + return createHash("sha256") + .update( + JSON.stringify([ + comparableSpec(draft.spec), + draft.followUp.investigateOnAttention, + draft.followUp.notifyExternally, + ]) + ) + .digest("hex"); +} + +/** The recorded outcome of the external consent, replayed rather than re-decided. */ +function recordedExternalNotification(recorded: WatchSubmission): WatchExternalNotification { + if (recorded.externalNotificationStatus === "enabled") return { status: "enabled" }; + if (recorded.externalNotificationStatus === "unavailable") { + return { status: "unavailable", reason: recorded.externalNotificationReason ?? "unknown" }; + } + return { status: "not_requested" }; +} + +/** The draft the ledger recorded, which is what a replay must be built from. */ +function recordedDraft(recorded: WatchSubmission, fallback: WatchDraft): WatchDraft { + const parsed = watchDraftSchema.safeParse(recorded.draft); + return parsed.success ? parsed.data : fallback; +} + +/** The refusal record, keyed off the request so a retry's success can still follow it. */ +function refusalMessage(clientRequestId: string, error: string): WatchTranscriptMessage { + return { + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}refused:${clientRequestId}`, + role: "assistant", + parts: [{ type: "text", text: error }], + }; +} + +/** + * Submit a configured watch card, for an already-authorized environment and a chat the caller + * owns. + * + * The submission ledger is the idempotency boundary, not the transcript ids: a row keyed + * `(chatId, clientRequestId)` is written *before* the condition is evaluated, and it carries + * the outcome once there is one. So a retry looks the submission up first and replays what + * was recorded — it never re-evaluates and never creates a second operation, even after the + * first watch has fired, expired or answered in one shot. Only a `pending` row, left by an + * attempt that died before writing its outcome, is allowed to proceed, and it converges on + * the watch id reserved up front rather than creating another. + * + * The transcript ordering is the second invariant: the record of what the user confirmed is + * written before anything starts running, and the confirmation after, so a crash can leave a + * watch that is visible but unconfirmed, never one that is live and invisible. + */ +export async function submitDashboardAgentWatch(params: { + environment: AuthenticatedEnvironment; + userId: string; + organizationId: string; + /** The chat the card was submitted from. A fresh panel has none, so one is created. */ + chatId?: string; + /** Stable per card submission: both transcript records are keyed off it. */ + clientRequestId: string; + draft: WatchDraft; + now?: Date; + deps?: { + create?: typeof createDashboardAgentWatch; + subscribe?: typeof subscribeUserToWatchAlerts; + } & NonNullable[0]["deps"]>; +}): Promise { + const { environment, userId, organizationId, clientRequestId, draft } = params; + const create = params.deps?.create ?? createDashboardAgentWatch; + const subscribe = params.deps?.subscribe ?? subscribeUserToWatchAlerts; + + const chatId = + params.chatId ?? + chatIdForRequest({ + organizationId, + userId, + environmentId: environment.id, + clientRequestId, + }); + if (!params.chatId) { + // Idempotent on the id, so a retry reuses the same chat rather than making another. + await createChat(dashboardAgentDb, { + id: chatId, + organizationId, + userId, + title: `Watch ${watchSubjectLabel(draft.spec)}`, + }); + } + + const digest = draftDigest(draft); + const request = requestMessage(clientRequestId, draft); + + /** Append-once, then return. Both records are deterministic, so a replay rewrites bytes. */ + const settle = async (args: { + confirmation: WatchTranscriptMessage; + watchId: string | null; + repaired: boolean; + }): Promise => { + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: args.confirmation, + }); + return { + ok: true, + chatId, + watching: args.watchId !== null, + watchId: args.watchId, + messages: [request, args.confirmation], + repaired: args.repaired, + }; + }; + + /** `confirmed` is the draft the confirmation speaks for: the recorded one on a replay. */ + const watchingConfirmation = (args: { + watchId: string; + unavailable: boolean; + external: WatchExternalNotification; + confirmed?: WatchDraft; + }) => { + const confirmed = args.confirmed ?? draft; + return confirmationMessage({ + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}${args.watchId}`, + blockId: args.watchId, + body: watchConfirmationBlockBody({ + spec: confirmed.spec, + watchId: args.watchId, + unavailable: args.unavailable, + followUp: { + investigateOnAttention: confirmed.followUp.investigateOnAttention, + external: args.external, + }, + }), + }); + }; + + const oneShotConfirmation = ( + result: "satisfied" | "terminal_unsatisfied", + confirmed: WatchDraft = draft + ) => + confirmationMessage({ + // No watch exists, so the request id is the only stable key for a one-shot. + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}one-shot:${clientRequestId}`, + blockId: watchIdentity(confirmed.spec), + body: watchOneShotBlockBody({ spec: confirmed.spec, result }), + }); + + /** + * Rebuild the transcript from a recorded outcome. Nothing is evaluated or created, and + * the record is built from the recorded draft, never the body of this attempt: the + * durable user message states what the first attempt confirmed. + */ + const replay = async (recorded: WatchSubmission): Promise => { + const confirmed = recordedDraft(recorded, draft); + + if (recorded.state === "created" && recorded.watchId) { + return settle({ + confirmation: watchingConfirmation({ + watchId: recorded.watchId, + unavailable: recorded.unavailable, + // Recorded, never re-decided: the confirmation already in the transcript is + // append-once, so a second decision here would contradict it forever. + external: recordedExternalNotification(recorded), + confirmed, + }), + watchId: recorded.watchId, + repaired: true, + }); + } + + if (recorded.state === "immediate") { + return settle({ + confirmation: oneShotConfirmation( + recorded.immediateResult === "satisfied" ? "satisfied" : "terminal_unsatisfied", + confirmed + ), + watchId: null, + repaired: true, + }); + } + + // Refused. Replayed verbatim, so the transcript and the response agree. + const error = recorded.refusalError ?? "That watch couldn't be started."; + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: refusalMessage(clientRequestId, error), + }); + return { + ok: false, + chatId, + code: (recorded.refusalCode as SubmitWatchErrorCode | null) ?? "internal", + error, + existingId: recorded.refusalExistingId, + }; + }; + + /** + * Record a refusal, then write it under the consent record rather than leaving it to a + * toast the reload forgets. Losing the write means another attempt already settled it. + */ + const refuse = async (refusal: { + code: SubmitWatchErrorCode; + error: string; + existingId?: string | null; + }): Promise => { + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "refused", + refusalCode: refusal.code, + refusalError: refusal.error, + refusalExistingId: refusal.existingId ?? null, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") return replay(winner); + } + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: refusalMessage(clientRequestId, refusal.error), + }); + return { ok: false, chatId, ...refusal }; + }; + + // Step one, before the condition is even read: the ledger row. Its primary key is what + // makes a retry a replay instead of a second operation. + const claim = await claimWatchSubmission(dashboardAgentDb, { + chatId, + clientRequestId, + organizationId, + userId, + projectId: environment.projectId, + environmentId: environment.id, + draftHash: digest, + draft: { spec: draft.spec, followUp: draft.followUp } as unknown as Record, + watchId: generateWatchId(), + }); + + // A chat can by design span environments, so a matching draft is not enough: the row + // has to have been written by this same tenancy, or a staging retry would replay a + // production watch. A mismatch is refused, never replayed. + const recordedScope = claim.submission; + if ( + recordedScope.organizationId !== organizationId || + recordedScope.userId !== userId || + recordedScope.projectId !== environment.projectId || + recordedScope.environmentId !== environment.id + ) { + return { + ok: false, + chatId, + code: "request_conflict", + error: "That request was already submitted somewhere else.", + }; + } + + // A different draft under the same request id is a different request, not a retry. + if (claim.submission.draftHash !== digest) { + return { + ok: false, + chatId, + code: "request_conflict", + error: "That request was already submitted with different settings.", + }; + } + + // Step two, before the watch can exist: a false here means the record is already there + // from an earlier attempt, and a deleted chat is caught by the create below. + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: request, + }); + + let submission = claim.submission; + + // A recorded outcome is replayed. A refusal produced no side effect, so it is the one + // state that may be attempted again — under a fresh reserved id, since the old one may + // already name a cancelled row. + if (submission.state === "refused") { + const reopened = await reopenWatchSubmission(dashboardAgentDb, { + chatId, + clientRequestId, + watchId: generateWatchId(), + }); + if (!reopened) { + const current = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (current && current.state !== "pending") return replay(current); + return refuse({ code: "internal", error: "That watch couldn't be started." }); + } + submission = reopened; + } else if (submission.state !== "pending") { + return replay(submission); + } + + /** Attach the channel, record the outcome, then confirm. A lost race replays the winner. */ + const settleCreated = async (args: { + watchId: string; + unavailable: boolean; + /** The watch was already there: this call adopted it rather than creating it. */ + adopted: boolean; + }): Promise => { + // Attached after the watch exists, and a failure here never fails the creation — it is + // said out loud in the confirmation instead, and recorded so a replay repeats it. + let external: WatchExternalNotification = { status: "not_requested" }; + if (draft.followUp.notifyExternally) { + const subscribed = await subscribe({ userId, environment }); + external = subscribed.ok + ? { status: "enabled" } + : { status: "unavailable", reason: subscribed.reason }; + } + + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "created", + watchId: args.watchId, + unavailable: args.unavailable, + external, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") { + // The winning outcome doesn't name this watch as created, so this watch is an orphan: + // cancel it before replaying, or a refusal would leave a live watch behind. + if (!(winner.state === "created" && winner.watchId === args.watchId)) { + await cancelWatch(dashboardAgentDb, { id: args.watchId, reason: "superseded" }); + } + return replay(winner); + } + } + + return settle({ + confirmation: watchingConfirmation({ + watchId: args.watchId, + unavailable: args.unavailable, + external, + }), + watchId: args.watchId, + repaired: args.adopted, + }); + }; + + // Converge: an attempt that died mid-create left its row under the reserved id. + const reservedWatchId = submission.watchId ?? generateWatchId(); + const reserved = await getWatch(dashboardAgentDb, { id: reservedWatchId }); + if (reserved) { + if (reserved.status === "cancelled") { + // The previous attempt created it and then took it back. The id is spent, so this + // submission can't be completed; a fresh submit gets a fresh request id. + return refuse({ + code: "internal", + error: "The watch couldn't be scheduled. Nothing is being watched.", + }); + } + // `unavailable` isn't recoverable here: it belonged to the attempt that died. + return settleCreated({ watchId: reserved.id, unavailable: false, adopted: true }); + } + + const result = await create({ + environment, + userId, + chatId, + spec: draft.spec, + investigateOnAttention: draft.followUp.investigateOnAttention, + watchId: reservedWatchId, + now: params.now, + deps: params.deps, + }); + + if (!result.ok) { + // Pre-ledger fallback: a submit that started before this ledger existed has no row of + // its own, so an active watch matching the draft is still adopted rather than refused. + // This only ever loads a watch; it never creates one. + if (result.code === "duplicate" && result.existingId) { + const existing = await getWatch(dashboardAgentDb, { id: result.existingId }); + if ( + existing && + existing.chatId === chatId && + existing.status === "active" && + isSameWatchRequest(existing, draft) + ) { + return settleCreated({ watchId: existing.id, unavailable: false, adopted: true }); + } + } + return refuse(result); + } + + if (!result.watching) { + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "immediate", + // No watch exists, so the reserved id is released rather than left dangling. + watchId: null, + immediateResult: result.immediate.result, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") return replay(winner); + } + return settle({ + confirmation: oneShotConfirmation( + result.immediate.result as "satisfied" | "terminal_unsatisfied" + ), + watchId: null, + repaired: false, + }); + } + + return settleCreated({ + watchId: result.watchId, + unavailable: result.unavailable === true, + adopted: false, + }); +} + +/** The two guardrail refusals, worded once for both the pre-check and the insert. */ +function creationGuardrailError( + refusal: + | { error: "limit_reached"; activeCount: number } + | { error: "duplicate"; existingId: string | null } +): CreateDashboardAgentWatchResult { + if (refusal.error === "limit_reached") { + return { + ok: false, + code: "limit_reached", + error: `This chat already has ${MAX_ACTIVE_WATCHES_PER_CHAT} active watches. Cancel one first.`, + }; + } + return { + ok: false, + code: "duplicate", + error: "This chat is already watching that.", + existingId: refusal.existingId, + }; +} + +/** + * Trigger one tick of the watcher task, as the agent's own environment. The token travels + * in the payload, not the database: signing is a pure function of the watch row. + */ +export async function scheduleWatchTick(params: { + watchId: string; + token: string; + delayMinutes: number; + /** The tick generation the scheduled invocation claims. */ + tick: number; +}): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + + await client.tasks.trigger( + WATCH_TASK_ID, + { watchId: params.watchId, token: params.token, apiOrigin, tick: params.tick }, + { + delay: `${params.delayMinutes}m`, + // Keyed on the generation the payload carries, so a retried schedule can't double-tick. + idempotencyKey: `watch:${params.watchId}:tick:${params.tick}`, + // Pin to the same deployed agent version the chat runs on, when set. + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** The task that polls a whole (environment, cadence) group. */ +export const WATCH_BATCH_TASK_ID = "dashboard-agent-watch-batch"; + +/** + * How long a chain may go silent before it is treated as dead and re-armed. Three cadences + * plus two minutes, so a tick's jitter and retries can't trip it. + */ +export function watchBatchStaleMs(cadenceMinutes: number): number { + return cadenceMinutes * 60_000 * 3 + 2 * 60_000; +} + +/** + * Make sure a chain is polling one (environment, cadence) group. A failed trigger un-arms the + * row, since a chain marked running with no run behind it leaves its group unpolled. + */ +export async function armDashboardAgentWatchBatch(params: { + environmentId: string; + cadenceMinutes: number; + now?: Date; + deps?: { + arm?: typeof armWatchBatch; + schedule?: typeof scheduleWatchBatchTick; + stop?: typeof stopWatchBatch; + }; +}): Promise<{ running: boolean }> { + const now = params.now ?? new Date(); + const arm = params.deps?.arm ?? armWatchBatch; + const schedule = params.deps?.schedule ?? scheduleWatchBatchTick; + const stop = params.deps?.stop ?? stopWatchBatch; + + const armed = await arm(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + staleBefore: new Date(now.getTime() - watchBatchStaleMs(params.cadenceMinutes)), + }); + + // A live chain already covers the group. + if (!armed) return { running: true }; + + try { + await schedule({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: armed.epoch, + // A claim lands on `generation + 1`. + tick: armed.generation + 1, + delayMinutes: params.cadenceMinutes, + }); + } catch (error) { + logger.error("Dashboard agent watch: failed to start a batch chain", { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + error, + }); + await stop(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: armed.epoch, + }); + return { running: false }; + } + + return { running: true }; +} + +/** + * Trigger one tick of a batch chain, as the agent's own environment. The chain's token names the + * group and nothing else; the batch check re-authorizes every watch against its own snapshot. + */ +export async function scheduleWatchBatchTick(params: { + environmentId: string; + cadenceMinutes: number; + epoch: number; + tick: number; + delayMinutes: number; +}): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + const token = await mintDashboardAgentWatchBatchToken({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + + await client.tasks.trigger( + WATCH_BATCH_TASK_ID, + { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + apiOrigin, + token, + epoch: params.epoch, + tick: params.tick, + }, + { + delay: `${params.delayMinutes}m`, + // The chain's own key shape, epoch included, so a re-armed chain can't collide + // with its predecessor's keys. + idempotencyKey: `watch-batch:${params.environmentId}:${params.cadenceMinutes}:${params.epoch}:tick:${params.tick}`, + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** + * Hand a resolved watch's wake to the watcher task, since only the agent project may append to a + * chat's `in` stream. Keyed per watch with a short TTL, so a later sweep can still retry. + */ +export async function scheduleWatchDelivery(watch: { id: string; expiresAt: Date }): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + const token = await mintDashboardAgentWatchToken({ + watchId: watch.id, + expiresAt: watch.expiresAt, + }); + + await client.tasks.trigger( + WATCH_TASK_ID, + { watchId: watch.id, token, apiOrigin, tick: 0, deliverOnly: true }, + { + idempotencyKey: `watch:${watch.id}:deliver`, + idempotencyKeyTTL: "10m", + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** + * Delete a chat and end its watches in one transaction, so no live watch is left on an + * invisible chat. Owner-scoped, so a chatId the caller doesn't own deletes nothing. + */ +export async function deleteChatWithWatches(params: { + chatId: string; + userId: string; +}): Promise<{ deleted: boolean; cancelledWatches: number }> { + const result = await softDeleteChat(dashboardAgentDb, params); + return { deleted: result.deleted, cancelledWatches: result.cancelledWatches.length }; +} + +/** Dates are strings because this crosses a loader's JSON boundary. */ +export type ChatWatchChip = { + id: string; + identity: string; + status: WatchStatus; + kind: string; + note: string; + checkEveryMinutes: number; + expiresAt: string; + endedReason: string | null; + /** How the watch ended. Null while active. */ + resolution: WatchResolution | null; + /** What the resolving check observed. */ + observedOutcome: WatchObservedOutcome | null; +}; + +/** + * Active watches for many chats in one query, keyed by chatId. The query layer re-scopes the + * chat ids by org and user, so this is safe with ids from any source. + */ +export async function listActiveWatchesForChats(params: { + chatIds: string[]; + organizationId: string; + userId: string; +}): Promise> { + const byChat = await listActiveWatchesForChatsQuery(dashboardAgentDb, params); + + return Object.fromEntries( + Object.entries(byChat).map(([chatId, watches]) => [ + chatId, + watches.map((watch) => ({ + id: watch.id, + identity: watch.identity, + status: watch.status, + kind: watch.kind, + note: watch.note, + checkEveryMinutes: watch.checkEveryMinutes, + expiresAt: watch.expiresAt.toISOString(), + endedReason: watch.endedReason, + resolution: watch.resolution, + observedOutcome: watch.observedOutcome, + })), + ]) + ); +} + +export function chatBelongsToUser(params: { + chatId: string; + userId: string; + organizationId: string; +}): Promise { + return chatExists(dashboardAgentDb, params); +} + +export type { ChatWatchContext }; + +/** + * Ownership check for a chat, plus its org, the tenancy floor its watches can't leave. No + * project or environment: those come from the authorized request context. + */ +export function resolveChatWatchContext(params: { + chatId: string; + userId: string; +}): Promise { + return getChatWatchContext(dashboardAgentDb, params); +} diff --git a/apps/webapp/app/v3/alertsWorker.server.ts b/apps/webapp/app/v3/alertsWorker.server.ts index 88637d1c361..a58426821e6 100644 --- a/apps/webapp/app/v3/alertsWorker.server.ts +++ b/apps/webapp/app/v3/alertsWorker.server.ts @@ -5,11 +5,37 @@ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; import { DeliverAlertService } from "./services/alerts/deliverAlert.server"; +import { + DeliverDashboardAgentWatchAlertService, + DeliverDashboardAgentWatchChannelAlertService, +} from "./services/alerts/deliverDashboardAgentWatchAlert.server"; import { DeliverErrorGroupAlertService } from "./services/alerts/deliverErrorGroupAlert.server"; import { ErrorAlertEvaluator } from "./services/alerts/errorAlertEvaluator.server"; +import { + watchObservedOutcomeSchema, + watchResolutionSchema, +} from "@internal/dashboard-agent-contracts"; import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server"; import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server"; +/** The fired watch, as the fan-out and each per-channel delivery carry it. */ +const DashboardAgentWatchAlertPayload = z.object({ + watchId: z.string(), + organizationId: z.string(), + projectId: z.string(), + environmentId: z.string(), + userId: z.string(), + identity: z.string(), + kind: z.string(), + note: z.string(), + firedAt: z.string(), + facts: z.record(z.unknown()), + // Optional so a job enqueued before this deploy still validates and delivers. + resolution: watchResolutionSchema.optional().catch(undefined), + // `.catch` so an unrecognized observation shape degrades instead of dropping the alert. + observed: watchObservedOutcomeSchema.optional().catch(undefined), +}); + function initializeWorker() { const redisOptions = { keyPrefix: "alerts:worker:", @@ -93,6 +119,24 @@ function initializeWorker() { }, logErrors: true, }, + // The fan-out: resolves the channels and enqueues one delivery job each. + "v3.deliverDashboardAgentWatchAlert": { + schema: DashboardAgentWatchAlertPayload, + visibilityTimeoutMs: 60_000, + retry: { + maxAttempts: 3, + }, + logErrors: true, + }, + // One channel's delivery, so a retry only re-sends the channel that failed. + "v3.deliverDashboardAgentWatchAlertChannel": { + schema: DashboardAgentWatchAlertPayload.extend({ channelId: z.string() }), + visibilityTimeoutMs: 60_000, + retry: { + maxAttempts: 3, + }, + logErrors: true, + }, }, concurrency: { workers: env.ALERTS_WORKER_CONCURRENCY_WORKERS, @@ -126,6 +170,14 @@ function initializeWorker() { const service = new DeliverErrorGroupAlertService(); await service.call(payload); }, + "v3.deliverDashboardAgentWatchAlert": async ({ payload }) => { + const service = new DeliverDashboardAgentWatchAlertService(); + await service.call(payload); + }, + "v3.deliverDashboardAgentWatchAlertChannel": async ({ payload }) => { + const service = new DeliverDashboardAgentWatchChannelAlertService(); + await service.call(payload); + }, }, }); diff --git a/apps/webapp/app/v3/commonWorker.server.ts b/apps/webapp/app/v3/commonWorker.server.ts index 65b59209ec5..f104cb4f15d 100644 --- a/apps/webapp/app/v3/commonWorker.server.ts +++ b/apps/webapp/app/v3/commonWorker.server.ts @@ -13,6 +13,10 @@ import { } from "~/services/attio.server"; import { sweepDashboardAgentTurnEvals } from "~/services/dashboardAgentEvalRetention.server"; import { sweepDashboardAgentInvestigations } from "~/services/dashboardAgentInvestigationSweep.server"; +import { + rearmDashboardAgentWatchBatches, + sweepDashboardAgentWatches, +} from "~/services/dashboardAgentWatchSweep.server"; import { logger } from "~/services/logger.server"; import { MembershipDevEnvironmentsSchema, @@ -158,6 +162,16 @@ function initializeWorker() { maxAttempts: 1, }, }, + // The watch backstops: expiry, wake redelivery, retention and dead batch chains. + "dashboardAgent.watchMaintenance": { + schema: CronSchema, + visibilityTimeoutMs: 60_000 * 5, + cron: "*/5 * * * *", + jitterInMs: 30_000, + retry: { + maxAttempts: 1, + }, + }, }, concurrency: { workers: env.COMMON_WORKER_CONCURRENCY_WORKERS, @@ -239,6 +253,30 @@ function initializeWorker() { failure ??= error; } + if (failure) throw failure; + }, + "dashboardAgent.watchMaintenance": async () => { + // Each backstop runs independently; the first failure is rethrown at the end. + let failure: unknown; + + try { + const watches = await sweepDashboardAgentWatches(); + if (watches.overdue > 0 || watches.undelivered > 0 || watches.purged > 0) { + logger.debug("Dashboard agent watch sweep", watches); + } + } catch (error) { + failure ??= error; + } + + try { + const batches = await rearmDashboardAgentWatchBatches(); + if (batches.stale > 0) { + logger.debug("Dashboard agent watch batch re-arm", batches); + } + } catch (error) { + failure ??= error; + } + if (failure) throw failure; }, }, diff --git a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts index 20d7c02333a..3ac7f4afb34 100644 --- a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts +++ b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts @@ -391,7 +391,8 @@ export class DeliverAlertService extends BaseService { break; } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } @@ -747,7 +748,8 @@ export class DeliverAlertService extends BaseService { break; } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } @@ -1024,7 +1026,8 @@ export class DeliverAlertService extends BaseService { return; } } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } diff --git a/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts new file mode 100644 index 00000000000..5c4e7a75a3d --- /dev/null +++ b/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts @@ -0,0 +1,506 @@ +import { + type ChatPostMessageArguments, + ErrorCode, + type WebAPIPlatformError, + type WebAPIRateLimitedError, +} from "@slack/web-api"; +import type { WatchObservedOutcome, WatchResolution } from "@internal/dashboard-agent-contracts"; +import { type ProjectAlertChannel } from "@trigger.dev/database"; +import assertNever from "assert-never"; +import { subtle } from "crypto"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { + isIntegrationForService, + type OrganizationIntegrationForService, + OrgIntegrationRepository, +} from "~/models/orgIntegration.server"; +import { + ProjectAlertEmailProperties, + ProjectAlertSlackProperties, + ProjectAlertWebhookProperties, +} from "~/models/projectAlert.server"; +import { mintDashboardAgentAlertUnsubscribeToken } from "~/services/dashboardAgentAlertUnsubscribeToken.server"; +import { + canUseDashboardAgentAlerts, + DASHBOARD_AGENT_WATCH_ALERT_TYPE, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { + presentResolvedWatch, + renderFactLines, + watchNoteLine, +} from "~/presenters/v3/dashboardAgent"; +import { sendAlertEmail } from "~/services/email.server"; +import { logger } from "~/services/logger.server"; +import { decryptSecret } from "~/services/secrets/secretStore.server"; +import { v3RunsPath } from "~/utils/pathBuilder"; +import { alertsWorker } from "~/v3/alertsWorker.server"; +import { safeWebhookFetch } from "./safeWebhookFetch.server"; + +/** + * Deliver a fired dashboard-agent watch to the project's alert channels. No `ProjectAlert` row: + * the job ids are the dedupe, and one job per channel keeps a failing webhook to itself. + */ +export type DashboardAgentWatchAlertPayload = { + watchId: string; + organizationId: string; + projectId: string; + environmentId: string; + userId: string; + identity: string; + kind: string; + note: string; + firedAt: string; + facts: Record; + /** Optional so a job from an older build still delivers, falling back to `condition_met`. */ + resolution?: WatchResolution; + observed?: WatchObservedOutcome; +}; + +/** One channel's delivery: the fan-out payload plus the channel it targets. */ +export type DashboardAgentWatchChannelAlertPayload = DashboardAgentWatchAlertPayload & { + channelId: string; +}; + +/** Bumped when the webhook body's shape changes. */ +const WEBHOOK_VERSION = "2026-08-02"; + +/** Wording comes from the shared presenter, so every surface says the same sentence. */ +function presentAlert(payload: DashboardAgentWatchAlertPayload) { + return presentResolvedWatch({ + kind: payload.kind, + identity: payload.identity, + // Only a met condition fans out today; the fallback keeps an older payload + // from silently presenting as something else. + resolution: payload.resolution ?? "condition_met", + observed: payload.observed ?? null, + }); +} + +class SkipRetryError extends Error {} + +type ResolvedContext = { + environmentName: string; + environmentSlug: string; + organizationSlug: string; + organizationTitle: string; + projectName: string; + projectSlug: string; + projectRef: string; + dashboardLink: string; +}; + +type ResolvedEnvironment = NonNullable>>; + +function findEnvironment(payload: DashboardAgentWatchAlertPayload) { + return $replica.runtimeEnvironment.findFirst({ + where: { id: payload.environmentId, projectId: payload.projectId }, + select: { + type: true, + slug: true, + branchName: true, + project: { + select: { + name: true, + slug: true, + externalRef: true, + organization: { select: { slug: true, title: true } }, + }, + }, + }, + }); +} + +function buildContext(environment: ResolvedEnvironment): ResolvedContext { + return { + environmentName: environment.branchName ?? environment.slug, + environmentSlug: environment.slug, + organizationSlug: environment.project.organization.slug, + organizationTitle: environment.project.organization.title, + projectName: environment.project.name, + projectSlug: environment.project.slug, + projectRef: environment.project.externalRef, + dashboardLink: `${env.APP_ORIGIN}${v3RunsPath( + { slug: environment.project.organization.slug }, + { slug: environment.project.slug }, + { slug: environment.slug } + )}`, + }; +} + +/** The fan-out: gate the watch, then enqueue one delivery job per channel. */ +export class DeliverDashboardAgentWatchAlertService { + async call(payload: DashboardAgentWatchAlertPayload): Promise { + const environment = await findEnvironment(payload); + + if (!environment) { + logger.warn("[DeliverDashboardAgentWatchAlert] Environment not found", { + watchId: payload.watchId, + }); + return; + } + + // Gate at delivery time, not only at subscribe time, so a plan change or a revoked + // feature flag stops the alerts without anyone cleaning up channels. + const gate = await canUseDashboardAgentAlerts({ + userId: payload.userId, + organizationId: payload.organizationId, + organizationSlug: environment.project.organization.slug, + }); + if (!gate.allowed) { + logger.info("[DeliverDashboardAgentWatchAlert] Not allowed for this organization", { + watchId: payload.watchId, + reason: gate.reason, + }); + return; + } + + const channels = await $replica.projectAlertChannel.findMany({ + where: { + projectId: payload.projectId, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + environmentTypes: { has: environment.type }, + }, + select: { id: true }, + }); + + for (const channel of channels) { + await alertsWorker.enqueue({ + // Stable per channel, so a fan-out retry re-enqueues the same job ids + // rather than a second alert per channel. + id: `watch-alert:${payload.watchId}:channel:${channel.id}`, + job: "v3.deliverDashboardAgentWatchAlertChannel", + payload: { ...payload, channelId: channel.id }, + }); + } + } +} + +/** One channel's delivery. A retry here can only re-send this channel. */ +export class DeliverDashboardAgentWatchChannelAlertService { + async call(payload: DashboardAgentWatchChannelAlertPayload): Promise { + // Re-read the channel rather than trusting the fan-out's snapshot: an unsubscribe + // between fan-out and delivery should stop the alert. The primary, since the + // unsubscribe writes there and replica lag would send the mail anyway. + const channel = await prisma.projectAlertChannel.findFirst({ + where: { + id: payload.channelId, + projectId: payload.projectId, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + }, + }); + + if (!channel) { + logger.info("[DeliverDashboardAgentWatchAlert] Channel gone or unsubscribed", { + watchId: payload.watchId, + channelId: payload.channelId, + }); + return; + } + + const environment = await findEnvironment(payload); + + if (!environment) { + logger.warn("[DeliverDashboardAgentWatchAlert] Environment not found", { + watchId: payload.watchId, + }); + return; + } + + const context = buildContext(environment); + + try { + switch (channel.type) { + case "EMAIL": + await this.#sendEmail(channel, payload, context); + break; + case "SLACK": + await this.#sendSlack(channel, payload, context); + break; + case "WEBHOOK": + await this.#sendWebhook(channel, payload, context); + break; + default: + assertNever(channel.type); + } + } catch (error) { + if (error instanceof SkipRetryError) { + logger.warn("[DeliverDashboardAgentWatchAlert] Skipping retry", { + watchId: payload.watchId, + channelId: channel.id, + reason: error.message, + }); + return; + } + throw error; + } + } + + async #sendEmail( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const emailProperties = ProjectAlertEmailProperties.safeParse(channel.properties); + if (!emailProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse email properties", { + issues: emailProperties.error.issues, + }); + return; + } + + const token = await mintDashboardAgentAlertUnsubscribeToken({ + channelId: channel.id, + alertType: DASHBOARD_AGENT_WATCH_ALERT_TYPE, + }); + + const presentation = presentAlert(payload); + + await sendAlertEmail({ + email: "alert-dashboard-agent-watch", + to: emailProperties.data.email, + identity: payload.identity, + kind: payload.kind, + headline: presentation.headline, + tone: presentation.tone, + note: payload.note, + // The sentence that quotes the note, rendered by the shared presenter so the + // email and the Slack message say it the same way. + noteLine: watchNoteLine(payload.note) ?? undefined, + firedAt: payload.firedAt, + facts: factList(payload.facts), + dashboardLink: context.dashboardLink, + unsubscribeLink: `${env.APP_ORIGIN}/resources/dashboard-agent/alerts/${channel.id}/unsubscribe?token=${encodeURIComponent(token)}`, + organization: context.organizationTitle, + project: context.projectName, + environment: context.environmentName, + }); + } + + async #sendSlack( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const slackProperties = ProjectAlertSlackProperties.safeParse(channel.properties); + if (!slackProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse slack properties", { + issues: slackProperties.error.issues, + }); + return; + } + + const integration = slackProperties.data.integrationId + ? await prisma.organizationIntegration.findFirst({ + where: { + id: slackProperties.data.integrationId, + organizationId: payload.organizationId, + }, + include: { tokenReference: true }, + }) + : await prisma.organizationIntegration.findFirst({ + where: { service: "SLACK", organizationId: payload.organizationId }, + orderBy: { createdAt: "desc" }, + include: { tokenReference: true }, + }); + + if (!integration || !isIntegrationForService(integration, "SLACK")) { + logger.error("[DeliverDashboardAgentWatchAlert] Slack integration not found"); + return; + } + + await this.#postSlackMessage(integration, { + channel: slackProperties.data.channelId, + ...this.#buildSlackMessage(payload, context), + } as ChatPostMessageArguments); + } + + async #sendWebhook( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const webhookProperties = ProjectAlertWebhookProperties.safeParse(channel.properties); + if (!webhookProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse webhook properties", { + issues: webhookProperties.error.issues, + }); + return; + } + + const rawPayload = JSON.stringify({ + // Stable across attempts, so a receiver can dedupe a redelivery. Unlike the + // error-group webhook, which mints a nanoid per attempt. + id: `watch:${payload.watchId}:channel:${payload.channelId}`, + created: new Date(payload.firedAt), + webhookVersion: WEBHOOK_VERSION, + type: "alert.dashboard_agent_watch", + object: { + watch: { + id: payload.watchId, + identity: payload.identity, + kind: payload.kind, + note: payload.note, + // `outcome` keeps its two-value encoding for receivers that already parse it; + // `resolution` and `observed` carry the detail. + outcome: "fired", + resolution: payload.resolution ?? "condition_met", + observed: payload.observed ?? null, + firedAt: payload.firedAt, + facts: payload.facts, + }, + environment: { id: payload.environmentId, name: context.environmentName }, + organization: { + id: payload.organizationId, + slug: context.organizationSlug, + name: context.organizationTitle, + }, + project: { + id: payload.projectId, + ref: context.projectRef, + slug: context.projectSlug, + name: context.projectName, + }, + dashboardUrl: context.dashboardLink, + }, + }); + + const secret = await decryptSecret(env.ENCRYPTION_KEY, webhookProperties.data.secret); + const key = await subtle.importKey( + "raw", + Buffer.from(secret, "utf-8"), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"] + ); + const signature = await subtle.sign("HMAC", key, Buffer.from(rawPayload, "utf-8")); + + // Deliver via the SSRF-safe wrapper (see safeWebhookFetch.server.ts). + const response = await safeWebhookFetch(webhookProperties.data.url, { + method: "POST", + headers: { + "content-type": "application/json", + "x-trigger-signature-hmacsha256": Buffer.from(signature).toString("hex"), + }, + body: rawPayload, + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + logger.info("[DeliverDashboardAgentWatchAlert] Failed to send webhook", { + status: response.status, + url: webhookProperties.data.url, + }); + throw new Error(`Failed to send watch alert webhook to ${webhookProperties.data.url}`); + } + } + + async #postSlackMessage( + integration: OrganizationIntegrationForService<"SLACK">, + message: ChatPostMessageArguments + ) { + const client = await OrgIntegrationRepository.getAuthenticatedClientForIntegration( + integration, + { forceBotToken: true } + ); + + try { + return await client.chat.postMessage({ + ...message, + unfurl_links: false, + unfurl_media: false, + }); + } catch (error) { + if (isWebAPIRateLimitedError(error)) { + throw new Error("Slack rate limited"); + } + if (isWebAPIPlatformError(error)) { + const code = (error as WebAPIPlatformError).data.error; + if (code === "invalid_blocks" || code === "account_inactive") { + throw new SkipRetryError(`Slack: ${code}`); + } + throw new Error("Slack platform error"); + } + throw error; + } + } + + #buildSlackMessage( + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): { text: string; blocks: object[] } { + const facts = factList(payload.facts); + const { headline } = presentAlert(payload); + const noteLine = watchNoteLine(payload.note); + + return { + // The notification text is the plain-text rendering, so what a phone shows + // matches what the blocks below say. + text: [`${headline} [${context.environmentName}]`, noteLine, ...renderFactLines(facts)] + .filter(Boolean) + .join("\n"), + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: [`*${headline}* [${context.environmentName}]`, noteLine] + .filter(Boolean) + .join("\n"), + }, + }, + ...(facts.length > 0 + ? [ + { + type: "section", + fields: facts.slice(0, 10).map((fact) => ({ + type: "mrkdwn", + text: `*${fact.label}:*\n${fact.value}`, + })), + }, + ] + : []), + { + type: "actions", + elements: [ + { + type: "button", + text: { type: "plain_text", text: "Open dashboard" }, + url: context.dashboardLink, + style: "primary", + }, + ], + }, + ], + }; + } +} + +/** + * The check's facts, flattened for display. Generic because the bag is per-watch-kind and + * open-ended, and capped so a big bag can't blow up an email or a Slack block. + */ +function factList(facts: Record): Array<{ label: string; value: string }> { + return Object.entries(facts) + .filter(([, value]) => value !== null && value !== undefined && value !== "") + .slice(0, 12) + .map(([key, value]) => ({ + label: humanizeFactKey(key), + value: typeof value === "object" ? JSON.stringify(value).slice(0, 200) : String(value), + })); +} + +function humanizeFactKey(key: string): string { + const spaced = key.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/[_-]+/g, " "); + return spaced.charAt(0).toUpperCase() + spaced.slice(1); +} + +function isWebAPIPlatformError(error: unknown): error is WebAPIPlatformError { + return (error as WebAPIPlatformError).code === ErrorCode.PlatformError; +} + +function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError { + return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError; +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 6a8641994cf..7bde8e8b1f2 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -20,6 +20,7 @@ "db:seed": "tsx seed.ts", "db:seed:ai-spans": "tsx seed-ai-spans.mts", "db:seed:queue-metrics": "tsx seed-queue-metrics.mts", + "scenarios:watch": "tsx seed-watch-scenarios.mts", "upload:sourcemaps": "bash ./upload-sourcemaps.sh", "test": "vitest --no-file-parallelism", "eval:dev": "evalite watch" diff --git a/apps/webapp/seed-watch-scenarios.mts b/apps/webapp/seed-watch-scenarios.mts new file mode 100644 index 00000000000..0e37dda9e85 --- /dev/null +++ b/apps/webapp/seed-watch-scenarios.mts @@ -0,0 +1,868 @@ +/** + * Local watch scenario kit. Run on Node 20; `--help` lists the verbs. + * Walkthroughs: internal-packages/dashboard-agent/GUIDEBOOK.md + */ +import { ClickHouse, TASK_RUN_COLUMNS } from "@internal/clickhouse"; +import type { QueueMetricsRawV1Input } from "@internal/clickhouse"; +import { randomUUID } from "node:crypto"; +// oxlint-disable import/default -- deliberate CommonJS interop: this entry is ESM, +// tsx compiles the app's `.ts` to CommonJS, and an ESM importer reaches a CommonJS +// module's exports only through `default`. +import dbServer from "./app/db.server"; +import errorFingerprinting from "./app/utils/errorFingerprinting"; +import eventCommon from "./app/v3/eventRepository/common.server"; + +const { prisma } = dbServer; +const { calculateErrorFingerprint } = errorFingerprinting; +const { generateTraceId, generateSpanId } = eventCommon; + +const APP_ORIGIN = process.env.APP_ORIGIN ?? "http://localhost:3030"; + +const DEFAULT_RUN_SECONDS = 60; +const DEFAULT_FAIL_TASK = "slow-fail"; +const DEFAULT_SUCCEED_TASK = "slow-succeed"; + +/** One fixed shape: the fingerprint has to stay stable across runs. */ +const SCENARIO_ERROR = { + type: "BUILT_IN_ERROR", + name: "ProviderError", + message: "429 Too Many Requests (rate_limit_exceeded)", + stackTrace: `ProviderError: 429 Too Many Requests (rate_limit_exceeded) + at sendEmail (src/trigger/scenarioKit.ts:31:11) + at run (src/trigger/scenarioKit.ts:18:5)`, +}; +const SCENARIO_ERROR_FINGERPRINT = calculateErrorFingerprint(SCENARIO_ERROR); +const SCENARIO_TASK_ID = "scenario-kit-task"; +const SCENARIO_QUEUE_NAME = "scenario-kit"; + +function fail(message: string): never { + console.error(`\n${message}\n`); + process.exit(1); +} + +function parseFlags(argv: string[]): { positional: string[]; flags: Record } { + const positional: string[] = []; + const flags: Record = {}; + for (let i = 0; i < argv.length; i++) { + const token = argv[i]; + // `pnpm run