diff --git a/.changeset/quiet-hounds-shave.md b/.changeset/quiet-hounds-shave.md new file mode 100644 index 00000000000..20e32667ebb --- /dev/null +++ b/.changeset/quiet-hounds-shave.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +The current-worker API now reports each task's queue, so you can see which tasks write to a given queue. diff --git a/.server-changes/dashboard-agent.md b/.server-changes/dashboard-agent.md index 504bb70c4cf..e40ae2076b9 100644 --- a/.server-changes/dashboard-agent.md +++ b/.server-changes/dashboard-agent.md @@ -7,6 +7,8 @@ Meet the dashboard agent: a chat in every environment that answers questions abo **Investigate** on a failed run, an error, a backed-up queue or a run that hasn't started gets you a worked-through answer — what happened, why, and how to fix it, with every claim linked to the runs, errors and deploys behind it. +**Watch…** on a run, queue, error or the health report tells you when things change: a run finishes, a queue clears or grows past a number you pick, an error comes back, an environment recovers. The answer arrives in the chat and, if you want, by email, Slack or webhook — and the agent can look into bad news on its own. A watch reaches you on any browser you sign in from, without opening the chat first. + The health report reads the same everywhere — dashboard, terminal, editor. A very long chat keeps working: the agent summarises the earlier part and carries on. The agent's replies no longer show images. A sample of conversations is scored automatically so the agent keeps getting better. Only the score and a one-line summary are kept, never your messages, data or code, and we can switch it off for your organization on request. diff --git a/apps/webapp/app/components/dashboard-agent/ActionsBlock.tsx b/apps/webapp/app/components/dashboard-agent/ActionsBlock.tsx index 4ef3a276b74..d4089e51533 100644 --- a/apps/webapp/app/components/dashboard-agent/ActionsBlock.tsx +++ b/apps/webapp/app/components/dashboard-agent/ActionsBlock.tsx @@ -4,16 +4,20 @@ import type { } from "@internal/dashboard-agent-contracts"; import { Button } from "~/components/primitives/Buttons"; import { ChatActionsRow } from "./chat-layout"; -import { renderableActions } from "./view-actions"; +import { renderableActions, withoutWatchActions } from "./view-actions"; export function ActionsBlock({ block, onIntent, + dropWatch = false, }: { block: ActionsBlockPayload; onIntent?: (intent: AgentIntent) => void; + /** Set when an investigation card in the same answer already offers the watch. */ + dropWatch?: boolean; }) { - const renderable = renderableActions(block.actions); + const actions = dropWatch ? withoutWatchActions(block.actions) : block.actions; + const renderable = renderableActions(actions); if (!onIntent || renderable.length === 0) return null; return ( diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx index e932c4e79f5..93ad5c761d3 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx @@ -1,4 +1,4 @@ -import type { SuggestedPrompt } from "@internal/dashboard-agent-contracts"; +import type { SuggestedPrompt, WatchSpec } from "@internal/dashboard-agent-contracts"; import { useLocation } from "@remix-run/react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { @@ -6,6 +6,9 @@ import { ResizablePanel, ResizablePanelGroup, } from "~/components/primitives/Resizable"; +import { useEnvironment } from "~/hooks/useEnvironment"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; import { useShortcutKeys } from "~/hooks/useShortcutKeys"; import { DashboardAgentPanel } from "./DashboardAgentPanel"; import { @@ -20,18 +23,92 @@ import { readAgentFullscreen, writeAgentFullscreen, } from "./panel-layout"; +import { nextPendingTurnChatId } from "./pending-turn"; +import { nextVisibleChat } from "./unread-counts"; +import { startWakePolling } from "./wake-poll"; +import { shouldPollWakeFeed, subscribeWatchActivity } from "./watch-activity"; +import { + showWatchWakesSummaryToast, + showWatchWakeToast, + WAKE_TOAST_MAX_INDIVIDUAL, + type WatchWake, +} from "./WatchWakeToast"; + +const TOASTED_WAKES_STORAGE_KEY = "tdev:dashboard-agent:toasted-wakes"; + +// Shorter than the poll interval, so a stuck request is dropped before the next tick. +const UNREAD_REQUEST_TIMEOUT_MS = 30_000; /** `hasAccess` is a UI gate only; the resource routes enforce the same check server-side. */ export function DashboardAgent({ children, hasAccess = false, promotedPrompt, + /** From the page load: unread wakes waiting for this user, whatever this browser remembers. */ + initialUnreadWakes = 0, + initialUnreadWork = 0, + /** Also from the page load: a watch is running, so a wake can still arrive in this tab. */ + hasActiveWatches = false, }: { children: React.ReactNode; hasAccess?: boolean; promotedPrompt?: SuggestedPrompt; + initialUnreadWakes?: number; + /** Chats whose transcript moved on since their owner last looked. */ + initialUnreadWork?: number; + hasActiveWatches?: boolean; }) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + const actionPath = `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/dashboard-agent`; + const [open, setOpen] = useState(false); + // Seeded from the page load, so the launcher dot is right before the first poll answers. + const [unreadWakes, setUnreadWakes] = useState(initialUnreadWakes); + // Work that finished behind a closed panel. Counted server-side on page load and refreshed + // with the chat list; the wake poll doesn't carry it. + const [unreadWork, setUnreadWork] = useState(initialUnreadWork); + // A turn this tab started may finish after the panel closes; that is exactly the case the + // dot exists for, so the poll has to be running when it lands. + const [pendingTurnChatId, setPendingTurnChatId] = useState(null); + const handleTurnActivityChange = useCallback((chatId: string, active: boolean) => { + setPendingTurnChatId((current) => nextPendingTurnChatId(current, { chatId, active })); + }, []); + const toastedWakes = useRef(new Set()); + // The toast source is recent deliveries, not unread, so the dedupe must survive a reload. + useEffect(() => { + try { + const raw = window.localStorage.getItem(TOASTED_WAKES_STORAGE_KEY); + if (raw) for (const id of JSON.parse(raw) as string[]) toastedWakes.current.add(id); + } catch { + // Storage unavailable; the in-memory dedupe still applies. + } + }, []); + const rememberToasted = useCallback((watchId: string) => { + toastedWakes.current.add(watchId); + try { + // Newest ids only, so the key can't grow unbounded. + window.localStorage.setItem( + TOASTED_WAKES_STORAGE_KEY, + JSON.stringify([...toastedWakes.current].slice(-50)) + ); + } catch { + // Same as the read. + } + }, []); + // A wake in the on-screen chat toasts but must not light the dot. + const visibleChat = useRef(null); + + // Switching environment re-runs the layout loader but does not remount it, so the seeds + // above would keep the old environment's counts. + const seededEnvironment = useRef(environment.id); + useEffect(() => { + if (seededEnvironment.current === environment.id) return; + seededEnvironment.current = environment.id; + setUnreadWakes(initialUnreadWakes); + setUnreadWork(initialUnreadWork); + }, [environment.id, initialUnreadWakes, initialUnreadWork]); // Read lazily so SSR always renders the side panel. const [fullscreen, setFullscreen] = useState(readAgentFullscreen); @@ -57,17 +134,32 @@ export function DashboardAgent({ const [requestedMessage, setRequestedMessage] = useState< { text: string; seq: number } | undefined >(undefined); + // `seq` so the same chat can be asked for twice. + const [openChatRequest, setOpenChatRequest] = useState< + { chatId: string; seq: number } | undefined + >(undefined); + const [watchRequest, setWatchRequest] = useState<{ spec: WatchSpec; seq: number } | undefined>( + undefined + ); const setPanelOpen = useCallback((next: boolean) => { setOpen(next); // Pending requests must be dropped or a stale one re-applies on the next open. if (!next) { + visibleChat.current = null; setFullscreen(false); writeAgentFullscreen(false); setRequestedMessage(undefined); + setOpenChatRequest(undefined); + setWatchRequest(undefined); } }, []); + const openChat = useCallback((chatId: string) => { + setOpen(true); + setOpenChatRequest((current) => ({ chatId, seq: (current?.seq ?? 0) + 1 })); + }, []); + const openWith = useCallback((text: string) => { const trimmed = text.trim(); if (!trimmed) return; @@ -75,6 +167,106 @@ export function DashboardAgent({ setRequestedMessage((current) => ({ text: trimmed, seq: (current?.seq ?? 0) + 1 })); }, []); + const openWithWatch = useCallback((spec: WatchSpec) => { + setOpen(true); + setWatchRequest((current) => ({ spec, seq: (current?.seq ?? 0) + 1 })); + }, []); + + // Nothing to be woken about means nothing to poll for. The page load's unread count and + // active-watch flag are the ungated signals; the browser's own memory of a watch starts the + // poll without a reload. Once any says yes this tab keeps polling, so a wake reaches a tab + // that was open before the watch existed. + const [watching, setWatching] = useState(false); + useEffect(() => { + const sync = () => { + if ( + shouldPollWakeFeed({ + serverUnreadWakes: initialUnreadWakes, + serverHasActiveWatches: hasActiveWatches, + serverUnreadWork: initialUnreadWork, + turnInFlight: pendingTurnChatId !== null, + organizationId: organization.id, + }) + ) + setWatching(true); + }; + sync(); + return subscribeWatchActivity(sync); + }, [organization.id, initialUnreadWakes, hasActiveWatches, initialUnreadWork, pendingTurnChatId]); + + useEffect(() => { + if (!hasAccess || !watching) return; + + let cancelled = false; + const load = async () => { + try { + // Bounded, so one stuck request can't hold the poll's in-flight guard. + const res = await fetch(`${actionPath}?unread=1`, { + signal: AbortSignal.timeout(UNREAD_REQUEST_TIMEOUT_MS), + }); + if (!res.ok) return; + const data = (await res.json()) as { + unreadWakes?: number; + unreadWork?: number; + wakes?: WatchWake[]; + }; + if (cancelled) return; + // The wakes list carries read ones too, so only unread ones are subtracted. + const unreadInView = (data.wakes ?? []).filter( + (wake) => wake.unread && wake.chatId === visibleChat.current + ).length; + setUnreadWakes(Math.max(0, (data.unreadWakes ?? 0) - unreadInView)); + // A chat open in the panel is being read right now, so it isn't unread work. + setUnreadWork(Math.max(0, (data.unreadWork ?? 0) - (open && visibleChat.current ? 1 : 0))); + + const fresh = (data.wakes ?? []).filter((wake) => !toastedWakes.current.has(wake.watchId)); + for (const wake of fresh) rememberToasted(wake.watchId); + + if (fresh.length > WAKE_TOAST_MAX_INDIVIDUAL) { + showWatchWakesSummaryToast(fresh.length, () => setPanelOpen(true)); + } else { + for (const wake of [...fresh].reverse()) { + showWatchWakeToast(wake, openChat); + } + } + } catch { + // Try again next tick. + } + }; + + const stop = startWakePolling({ + load, + isHidden: () => document.hidden, + onVisibilityChange: (listener) => { + document.addEventListener("visibilitychange", listener); + return () => document.removeEventListener("visibilitychange", listener); + }, + }); + + return () => { + cancelled = true; + stop(); + }; + }, [hasAccess, watching, actionPath, setPanelOpen, openChat]); + + // Zeroes the wake dot right away; the poll restores the truth if another chat has one. The + // work count is not touched here: the panel derives it from the chat list. + const markChatRead = useCallback( + async (chatId: string, options: { leaving: boolean }) => { + visibleChat.current = nextVisibleChat(chatId, options); + setUnreadWakes(0); + const body = new FormData(); + body.set("intent", "read"); + body.set("chatId", chatId); + try { + await fetch(actionPath, { method: "POST", body }); + } catch { + // Catches up on the next open. + } + }, + [actionPath] + ); + // ⌘J is contextual: closed opens the panel, open starts a new chat. It never closes. useShortcutKeys({ shortcut: TOGGLE_PANEL_SHORTCUT, @@ -99,8 +291,8 @@ export function DashboardAgent({ useDashboardAgentOpenRequests({ enabled: hasAccess, openWith, setOpen: setPanelOpen }); const context = useMemo( - () => ({ open, setOpen: setPanelOpen, openWith }), - [open, setPanelOpen, openWith] + () => ({ open, setOpen: setPanelOpen, openWith, openWithWatch, unreadWakes, unreadWork }), + [open, setPanelOpen, openWith, openWithWatch, unreadWakes, unreadWork] ); if (!hasAccess) { @@ -129,8 +321,13 @@ export function DashboardAgent({ setPanelOpen(false)} requestedMessage={requestedMessage} + openChatRequest={openChatRequest} + watchRequest={watchRequest} newChatSeq={newChatSeq} promotedPrompt={promotedPrompt} + onChatRead={markChatRead} + onUnreadWorkChange={setUnreadWork} + onTurnActivityChange={handleTurnActivityChange} isFullscreen={fullscreen} onToggleFullscreen={toggleFullscreen} /> diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx index 526dedde342..11261222d5c 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx @@ -1,7 +1,12 @@ import { useChat } from "@ai-sdk/react"; import type { UIMessage } from "@ai-sdk/react"; import type { dashboardAgent } from "@internal/dashboard-agent"; -import type { AgentIntent, SuggestedPrompt } from "@internal/dashboard-agent-contracts"; +import { + isWatchRequestMessageId, + type AgentIntent, + type SuggestedPrompt, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; import { useNavigate } from "@remix-run/react"; import { useTriggerChatTransport } from "@trigger.dev/sdk/chat/react"; import { useCallback, useEffect, useRef, useState } from "react"; @@ -14,7 +19,7 @@ import { DashboardAgentMessages, type TurnActivity } from "./DashboardAgentMessa import { MESSAGE_TOO_LARGE_ERROR } from "./message-limits"; import { createTranscriptOrder, orderTranscript } from "./message-order"; import { appendRunFilters } from "./navigate-target"; -import { pendingNavigateIntents } from "./pending-intents"; +import { pendingNavigateIntents, pendingWatchIntents } from "./pending-intents"; import type { AgentPageContext } from "./page-context-types"; import { fetchChatTranscript, @@ -23,6 +28,7 @@ import { } from "./settled-transcript"; import { useAgentMessageQuota } from "./useAgentMessageQuota"; import { useTriggerUriResolver } from "./useTriggerUriResolver"; +import { WatchChips, type WatchChip } from "./WatchChips"; // Resuming with `lastEventId` stops the `.out` stream replaying the previous turn. export type DashboardAgentSession = { @@ -55,7 +61,12 @@ export function DashboardAgentChat({ streaming, prefill, promotedPrompt, + watches, pagePaths, + watchCard, + appendedMessages, + onWatchIntent, + onCancelWatch, onTurnSettled, onActivityChange, }: { @@ -75,7 +86,13 @@ export function DashboardAgentChat({ // `seq` makes each request distinct so the same text can be sent twice. prefill?: { text: string; seq: number }; promotedPrompt?: SuggestedPrompt; + watches: WatchChip[]; pagePaths?: Record; + watchCard?: React.ReactNode; + appendedMessages?: { messages: UIMessage[]; seq: number }; + /** Nothing is persisted until the user submits the card. */ + onWatchIntent?: (spec: WatchSpec) => void; + onCancelWatch: (watchId: string) => void; onTurnSettled: () => void; onActivityChange?: (chatId: string, activity: TurnActivity | null) => void; }) { @@ -172,6 +189,20 @@ export function DashboardAgentChat({ const activity: TurnActivity | null = status === "submitted" ? "thinking" : status === "streaming" ? "working" : null; + // Once per `seq`: the append is already persisted, so a replay would duplicate it. + // Ids are stable, so anything already in the transcript is skipped. + const appendedSeq = useRef(undefined); + useEffect(() => { + if (!appendedMessages || appendedSeq.current === appendedMessages.seq) return; + appendedSeq.current = appendedMessages.seq; + setMessages((current) => { + const missing = appendedMessages.messages.filter( + (message) => !current.some((existing) => existing.id === message.id) + ); + return missing.length === 0 ? current : [...current, ...missing]; + }); + }, [appendedMessages, setMessages]); + const sentFirst = useRef(false); useEffect(() => { if (pendingFirstMessage && !sentFirst.current) { @@ -192,7 +223,10 @@ export function DashboardAgentChat({ ); const retry = useCallback(() => { - const lastUserMessage = [...messages].reverse().find((m) => m.role === "user"); + // A watch's consent record is a user message nobody typed, so retry skips it. + const lastUserMessage = [...messages] + .reverse() + .find((m) => m.role === "user" && !isWatchRequestMessageId(m.id)); const text = lastUserMessage?.parts ?.filter((p): p is { type: "text"; text: string } => p.type === "text") .map((p) => p.text) @@ -230,6 +264,9 @@ export function DashboardAgentChat({ case "ask": submit(intent.prompt); return; + case "watch": + onWatchIntent?.(intent.spec); + return; case "navigate": void goTo(intent); return; @@ -237,7 +274,7 @@ export function DashboardAgentChat({ console.warn(`Dashboard agent: unhandled intent "${intent.kind}"`); } }, - [submit, goTo] + [submit, goTo, onWatchIntent] ); // Seeded from the loaded transcript before first render, so history never re-navigates. @@ -252,6 +289,17 @@ export function DashboardAgentChat({ if (target) void goTo(target); }, [messages, goTo]); + const watchProposedRef = useRef | null>(null); + if (watchProposedRef.current === null) { + watchProposedRef.current = new Set(); + pendingWatchIntents(initialMessages, watchProposedRef.current); + } + useEffect(() => { + const pending = pendingWatchIntents(messages, watchProposedRef.current!); + const proposed = pending.at(-1); + if (proposed) onWatchIntent?.(proposed.spec); + }, [messages, onWatchIntent]); + const stop = useCallback(() => { transport.stopGeneration(chatId); aiStop(); @@ -286,6 +334,10 @@ export function DashboardAgentChat({ return ( <> + watch.status === "active")} + onCancel={onCancelWatch} + /> {messages.length === 0 && !pendingFirstMessage ? ( )} + {watchCard ?
{watchCard}
: null} {quota.kind === "reached" ? ( void; projectSlug: string; @@ -22,6 +23,7 @@ export function DashboardAgentDraft({ currentPage: string; pageContext?: AgentPageContext; promotedPrompt?: SuggestedPrompt; + watchCard?: React.ReactNode; }) { const [input, setInput] = useState(""); @@ -56,6 +58,7 @@ export function DashboardAgentDraft({ promoted={promotedPrompt} composer={
+ {watchCard} submit(input)} onStop={() => {}} isStreaming={false} - placeholderSuggestion={placeholderSuggestion} + placeholderSuggestion={watchCard ? undefined : placeholderSuggestion} context={ void; }) { const [isHistoryOpen, setHistoryOpen] = useState(false); + const [pendingDelete, setPendingDelete] = useState(null); return (
@@ -76,11 +81,20 @@ export function DashboardAgentHeader({ setHistoryOpen(false); onSelectChat(chatId); }} - onDelete={onDeleteChat} + onRequestDelete={(chat) => { + setHistoryOpen(false); + setPendingDelete(chat); + }} /> + !open && setPendingDelete(null)} + onConfirm={onDeleteChat} + /> +
{showNewChat && ( - } - cancelButton={ - - } - /> -
- - - +// Rendered outside the history popover: inside it, focus moving to the dialog dismisses the +// popover, which unmounts the dialog before it can be answered. +export function DashboardAgentDeleteChatDialog({ + chat, + onOpenChange, + onConfirm, +}: { + chat: DashboardAgentChat | null; + onOpenChange: (open: boolean) => void; + onConfirm: (chatId: string) => void; +}) { + return ( + + + Delete this chat? +
+ + "{chat?.title}" and everything in it will be deleted. This can't be undone. + + { + if (chat) onConfirm(chat.id); + onOpenChange(false); + }} + > + Delete chat + + } + cancelButton={ + + } + /> +
+ + ); } diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx index 551902bba57..423beab7093 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx @@ -16,6 +16,7 @@ import { ChatText, ChatTranscript, ChatTurn, + ChatWakeSlot, } from "./chat-layout"; import { reuseWinners } from "./investigation-winners"; import { stripModelImages } from "./model-markdown"; @@ -24,6 +25,7 @@ import { shouldShowLiveTurnError } from "./turn-error"; import type { ResolvedUri } from "./ReportView"; import { answerContinuesAfter } from "./view-actions"; import { ViewBlocks } from "./view-catalog"; +import { findWakeWatch, WakeBanner, wakeRefFromMessageId, type WakeWatch } from "./WakeBanner"; export type { TurnActivity }; @@ -36,6 +38,8 @@ export type DashboardAgentMessagesProps = { onIntent?: (intent: AgentIntent) => void; resolveUri?: (uri: string) => ResolvedUri | null; pagePaths?: Record; + /** Optional: without it a wake banner falls back to kind-agnostic wording. */ + watches?: WakeWatch[]; }; // Returns the same reference when there are no `step-start` parts, so memoization holds. @@ -202,12 +206,14 @@ const DashboardAgentTurn = memo(function DashboardAgentTurn({ onIntent, resolveUri, pagePaths, + watches, investigationWinners, }: { message: UIMessage; onIntent?: (intent: AgentIntent) => void; resolveUri?: (uri: string) => ResolvedUri | null; pagePaths?: Record; + watches?: WakeWatch[]; /** See {@link winningInvestigationOccurrences}. */ investigationWinners?: Map; }) { @@ -266,6 +272,21 @@ const DashboardAgentTurn = memo(function DashboardAgentTurn({ body.push(renderDashboardPart(part, i, resolveUri)); } + const wake = wakeRefFromMessageId(message.id); + if (wake) { + return ( + + + } + > + {body} + + + ); + } + return {body}; }); @@ -278,6 +299,7 @@ export function DashboardAgentTurns({ onIntent, resolveUri, pagePaths, + watches, }: DashboardAgentMessagesProps) { // Must be the exact parts the turns render: the winners map keys by part index. const stripped = useMemo(() => messages.map(stripStepParts), [messages]); @@ -298,6 +320,7 @@ export function DashboardAgentTurns({ onIntent={onIntent} resolveUri={resolveUri} pagePaths={pagePaths} + watches={watches} investigationWinners={investigationWinners} /> ))} diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx index 64c14afc68c..0f54f72c652 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx @@ -1,7 +1,7 @@ import type { UIMessage } from "@ai-sdk/react"; import { useLocation } from "@remix-run/react"; import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from "react"; import { AgentSpinner } from "~/components/primitives/Spinner"; import { useToast } from "~/components/primitives/Toast"; import { useAgentPageContext } from "~/hooks/useAgentPageContext"; @@ -16,12 +16,18 @@ import { type DashboardAgentSession, } from "./DashboardAgentChat"; import { DashboardAgentDraft } from "./DashboardAgentDraft"; +import { WatchCard } from "./WatchCard"; +import { watchDraftFor } from "./watch-card"; +import { NO_WATCH_CARD, watchCardReducer } from "./watch-card-state"; +import { forgetWatchActivity, rememberWatchActivity } from "./watch-activity"; import type { TurnActivity } from "./DashboardAgentMessages"; import { DashboardAgentHeader } from "./DashboardAgentHeader"; import type { DashboardAgentChat as DashboardAgentChatListItem } from "./DashboardAgentHistory"; -import type { SuggestedPrompt } from "@internal/dashboard-agent-contracts"; +import type { SuggestedPrompt, WatchSpec } from "@internal/dashboard-agent-contracts"; import type { AgentPageContext } from "./page-context-types"; import { agentPageLabel } from "./page-label"; +import { escapeClosesPanel } from "./panel-escape"; +import { markChatListRead, unreadWorkCount } from "./unread-counts"; import { AgentPanelColumn } from "./panel-layout"; import { concurrencyPath } from "~/utils/pathBuilder"; @@ -63,8 +69,13 @@ type ActiveChat = { export function DashboardAgentPanel({ onClose, requestedMessage, + openChatRequest, newChatSeq, promotedPrompt, + watchRequest, + onChatRead, + onUnreadWorkChange, + onTurnActivityChange, isFullscreen = false, onToggleFullscreen, }: { @@ -73,8 +84,15 @@ export function DashboardAgentPanel({ onToggleFullscreen?: () => void; // Every `seq` below distinguishes repeat requests with identical contents. requestedMessage?: { text: string; seq: number }; + openChatRequest?: { chatId: string; seq: number }; newChatSeq?: number; promotedPrompt?: SuggestedPrompt; + watchRequest?: { spec: WatchSpec; seq: number }; + onChatRead?: (chatId: string, options: { leaving: boolean }) => void; + /** How many chats still hold work their owner hasn't seen. */ + onUnreadWorkChange?: (count: number) => void; + /** Whether a turn is running in a chat, so a closed panel still knows to expect an answer. */ + onTurnActivityChange?: (chatId: string, active: boolean) => void; }) { const organization = useOrganization(); const project = useProject(); @@ -88,7 +106,12 @@ export function DashboardAgentPanel({ const actionPath = `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/dashboard-agent`; const storageKey = lastChatStorageKey(organization.id); + const panelRef = useRef(null); + // Declared before the chat plumbing: changing chat dispatches into it. + const [watchCard, dispatchWatchCard] = useReducer(watchCardReducer, NO_WATCH_CARD); const [chats, setChats] = useState([]); + // Until the list has arrived, the page load's server count is the better answer. + const [chatsLoaded, setChatsLoaded] = useState(false); const [active, setActive] = useState(null); // Starts true so an `openWith` request waits for the restore instead of racing it. const [loading, setLoading] = useState( @@ -118,14 +141,21 @@ export function DashboardAgentPanel({ ); const [thinkingChatId, setThinkingChatId] = useState(null); - const handleActivityChange = useCallback((chatId: string, activity: TurnActivity | null) => { - setThinkingChatId((previous) => - activity !== null ? chatId : previous === chatId ? null : previous - ); - }, []); + const handleActivityChange = useCallback( + (chatId: string, activity: TurnActivity | null) => { + setThinkingChatId((previous) => + activity !== null ? chatId : previous === chatId ? null : previous + ); + onTurnActivityChange?.(chatId, activity !== null); + }, + [onTurnActivityChange] + ); const historyInFlight = useRef | null>(null); + // The read POST and its reload can land out of order, so mask the next list. + const justRead = useRef>(new Set()); + const loadHistory = useCallback(async () => { if (historyInFlight.current) return historyInFlight.current; const request = (async () => { @@ -133,7 +163,19 @@ export function DashboardAgentPanel({ const res = await fetch(actionPath); if (!res.ok) throw new Error(`History request failed (${res.status})`); const data = (await res.json()) as { chats?: DashboardAgentChatListItem[] }; - setChats(data.chats ?? []); + const read = justRead.current; + justRead.current = new Set(); + const chats = data.chats ?? []; + // Reloaded after every turn and after a watch is created, so this is where the browser + // learns whether the wake feed is worth polling. + const pending = chats.some((chat) => chat.hasActiveWatch || chat.hasUnreadWake); + if (pending) rememberWatchActivity(organization.id); + else forgetWatchActivity(organization.id); + const settled = chats.map((chat) => + read.has(chat.id) ? { ...chat, hasUnreadWake: false, hasUnreadWork: false } : chat + ); + setChats(settled); + setChatsLoaded(true); } catch (error) { console.error("Dashboard agent: failed to load chat history", error); toast.error("We couldn't load your previous chats. Try again in a moment."); @@ -143,14 +185,21 @@ export function DashboardAgentPanel({ })(); historyInFlight.current = request; return request; - }, [actionPath, toast]); + }, [actionPath, organization.id, toast]); // Bumped on each open so a slower earlier open can't overwrite a newer one. const openChatRequestSeq = useRef(0); + // The one way the panel changes chat: it invalidates any in-flight open and abandons a + // half-configured watch card, which would otherwise be submitted against the new chat. + const claimChatSlot = useCallback(() => { + dispatchWatchCard({ type: "chat-changed" }); + return ++openChatRequestSeq.current; + }, []); + const openChat = useCallback( async (id: string) => { - const seq = ++openChatRequestSeq.current; + const seq = claimChatSlot(); setLoading(true); try { const res = await fetch(`${actionPath}?chatId=${encodeURIComponent(id)}`); @@ -187,12 +236,12 @@ export function DashboardAgentPanel({ if (seq === openChatRequestSeq.current) setLoading(false); } }, - [actionPath, toast] + [actionPath, claimChatSlot, toast] ); const createChat = useCallback( async (text: string) => { - const seq = ++openChatRequestSeq.current; + const seq = claimChatSlot(); setLoading(true); try { const userMessage: UIMessage = { @@ -233,7 +282,7 @@ export function DashboardAgentPanel({ if (seq === openChatRequestSeq.current) setLoading(false); } }, - [actionPath, clientData, toast] + [actionPath, claimChatSlot, clientData, toast] ); const restored = useRef(false); @@ -256,12 +305,24 @@ export function DashboardAgentPanel({ useEffect(() => { if (panelOrg.current === organization.id) return; panelOrg.current = organization.id; - openChatRequestSeq.current += 1; + claimChatSlot(); setActive(null); setLoading(false); setChats([]); + setChatsLoaded(false); void loadHistory(); - }, [organization.id, loadHistory]); + }, [organization.id, claimChatSlot, loadHistory]); + + const handledOpenChatSeq = useRef(undefined); + useEffect(() => { + if (!openChatRequest || handledOpenChatSeq.current === openChatRequest.seq) return; + handledOpenChatSeq.current = openChatRequest.seq; + // Reloading the visible transcript would drop a turn in flight. + if (openChatRequest.chatId === active?.chatId) return; + void openChat(openChatRequest.chatId); + // `active` is read, not tracked: a later change must not re-run the request. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [openChatRequest, openChat]); useEffect(() => { if (!active?.chatId) return; @@ -275,6 +336,26 @@ export function DashboardAgentPanel({ } }, [active?.chatId, storageKey, location.pathname]); + useEffect(() => { + if (!active?.chatId) return; + const chatId = active.chatId; + onChatRead?.(chatId, { leaving: false }); + justRead.current.add(chatId); + setChats((previous) => markChatListRead(previous, chatId)); + // Read again on the way out: a wake can land while the chat is open. + return () => { + onChatRead?.(chatId, { leaving: true }); + justRead.current.add(chatId); + setChats((previous) => markChatListRead(previous, chatId)); + }; + }, [active?.chatId, onChatRead]); + + // The one source for the dot's work count: nudging it per open double-subtracts. + useEffect(() => { + if (!chatsLoaded) return; + onUnreadWorkChange?.(unreadWorkCount(chats)); + }, [chats, chatsLoaded, onUnreadWorkChange]); + // Bound to its chat, which remounts with a fresh guard ref on every switch. const [prefill, setPrefill] = useState<{ text: string; seq: number; chatId: string } | undefined>( undefined @@ -291,12 +372,107 @@ export function DashboardAgentPanel({ } }, [requestedMessage, loading, active, createChat]); + // Carries its chat id so a later-mounted chat cannot adopt another chat's block. + const [appendedMessages, setAppendedMessages] = useState< + { chatId: string; messages: UIMessage[]; seq: number } | undefined + >(undefined); + + const handledWatchSeq = useRef(undefined); + useEffect(() => { + if (!watchRequest || handledWatchSeq.current === watchRequest.seq) return; + handledWatchSeq.current = watchRequest.seq; + dispatchWatchCard({ + type: "open", + draft: watchDraftFor(watchRequest.spec), + requestId: generateFriendlyId("wreq"), + }); + }, [watchRequest]); + + // Nothing is posted or persisted until the card is submitted. + const openWatchCard = useCallback((spec: WatchSpec) => { + dispatchWatchCard({ + type: "open", + draft: watchDraftFor(spec), + requestId: generateFriendlyId("wreq"), + }); + }, []); + + const dismissWatchCard = useCallback(() => dispatchWatchCard({ type: "dismissed" }), []); + + const submitWatch = useCallback(async () => { + const draft = watchCard.draft; + if (!draft) return; + // Held across retries, so a resubmit repairs the same pair of records. + const clientRequestId = watchCard.requestId ?? generateFriendlyId("wreq"); + dispatchWatchCard({ type: "submitting", requestId: clientRequestId }); + try { + const body = new FormData(); + body.set("intent", "watch-create"); + body.set("draft", JSON.stringify(draft)); + body.set("clientRequestId", clientRequestId); + // A watch is chat-bound: with no chat open the server creates one. + if (active?.chatId) body.set("chatId", active.chatId); + + const res = await fetch(actionPath, { method: "POST", body }); + const data = (await res.json()) as { + chatId?: string; + messages?: UIMessage[]; + error?: string; + }; + if (!res.ok || !data.chatId || !data.messages) { + dispatchWatchCard({ + type: "failed", + error: data.error ?? "We couldn't start that watch. Try again in a moment.", + }); + return; + } + + const messages = data.messages; + if (active?.chatId === data.chatId) { + setAppendedMessages((current) => ({ + chatId: data.chatId!, + messages, + seq: (current?.seq ?? 0) + 1, + })); + dispatchWatchCard({ type: "submitted" }); + } else { + claimChatSlot(); + // No session: nothing is streaming and the records are the whole chat. + setActive({ chatId: data.chatId, messages, session: null }); + } + void loadHistory(); + } catch (error) { + console.error("Dashboard agent: failed to create watch", error); + dispatchWatchCard({ + type: "failed", + error: "We couldn't start that watch. Try again in a moment.", + }); + } + }, [ + watchCard.draft, + watchCard.requestId, + active?.chatId, + actionPath, + claimChatSlot, + loadHistory, + ]); + + const watchCardElement = watchCard.draft ? ( + dispatchWatchCard({ type: "edit", draft })} + onSubmit={() => void submitWatch()} + onCancel={dismissWatchCard} + pending={watchCard.pending} + error={watchCard.error} + /> + ) : null; + const newChat = useCallback(() => { - // Invalidate any in-flight open or create so its result can't replace the draft. - openChatRequestSeq.current += 1; + claimChatSlot(); setLoading(false); setActive(null); - }, []); + }, [claimChatSlot]); const switchChat = useCallback( (id: string) => { @@ -333,16 +509,54 @@ export function DashboardAgentPanel({ [actionPath, active?.chatId, newChat, loadHistory, toast] ); + const cancelWatch = useCallback( + async (watchId: string) => { + const chatId = active?.chatId; + if (!chatId) return; + setChats((previous) => + previous.map((chat) => + chat.id === chatId + ? { ...chat, watches: (chat.watches ?? []).filter((watch) => watch.id !== watchId) } + : chat + ) + ); + const body = new FormData(); + body.set("intent", "watch-cancel"); + body.set("chatId", chatId); + body.set("watchId", watchId); + try { + const res = await fetch(actionPath, { method: "POST", body }); + if (!res.ok) throw new Error(`Watch cancel failed (${res.status})`); + } catch (error) { + console.error("Dashboard agent: failed to cancel watch", error); + toast.error("We couldn't stop that watch. Try again in a moment."); + } + void loadHistory(); + }, + [actionPath, active?.chatId, loadHistory, toast] + ); + // Titles are written when the first turn settles, so a new chat has none yet. const activeChat = active ? chats.find((chat) => chat.id === active.chatId) : undefined; const headerTitle = active ? (activeChat?.title ?? "Chat") : "New chat"; + // Not filtered to active: the wake banner needs watches that already fired. + const chatWatches = activeChat?.watches ?? []; + return (
{ - if (event.key !== "Escape" || event.defaultPrevented) return; + if ( + !escapeClosesPanel({ + key: event.key, + defaultPrevented: event.defaultPrevented, + targetInsidePanel: panelRef.current?.contains(event.target as Node) ?? false, + }) + ) + return; event.preventDefault(); onClose(); }} @@ -384,7 +598,14 @@ export function DashboardAgentPanel({ environmentSlug={environment.slug} currentPage={currentPage} promotedPrompt={promotedPrompt} + watches={chatWatches} pagePaths={pagePaths} + watchCard={watchCardElement} + appendedMessages={ + appendedMessages?.chatId === active.chatId ? appendedMessages : undefined + } + onWatchIntent={openWatchCard} + onCancelWatch={cancelWatch} // The generated chat name is written before the turn-complete chunk lands. onTurnSettled={loadHistory} onActivityChange={handleActivityChange} @@ -397,6 +618,7 @@ export function DashboardAgentPanel({ currentPage={currentPage} pageContext={pageContext} promotedPrompt={promotedPrompt} + watchCard={watchCardElement} /> )} diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx index c7f935b1d9b..b9a53192ba9 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx @@ -1,6 +1,7 @@ import { BookOpenIcon, ChartBarIcon, + EyeIcon, MagnifyingGlassIcon, QuestionMarkCircleIcon, SparklesIcon, @@ -22,6 +23,7 @@ export const PROMPT_SLOT_BUTTON: Record< > = { promoted: { variant: "primary/small", icon: SparklesIcon }, investigate: { variant: "primary/small", icon: MagnifyingGlassIcon }, + watch: { variant: "secondary/small", icon: EyeIcon }, status: { variant: "secondary/small", icon: ChartBarIcon }, explain: { variant: "tertiary/small", icon: QuestionMarkCircleIcon }, docs: { variant: "docs/small", icon: BookOpenIcon }, diff --git a/apps/webapp/app/components/dashboard-agent/ReportView.tsx b/apps/webapp/app/components/dashboard-agent/ReportView.tsx index 936d7b319da..562cbe40430 100644 --- a/apps/webapp/app/components/dashboard-agent/ReportView.tsx +++ b/apps/webapp/app/components/dashboard-agent/ReportView.tsx @@ -31,6 +31,7 @@ import { import { type ReportMessages } from "~/presenters/v3/reports/report-messages"; import { AgentBadge } from "./agent-badges"; import { + FOOTER_WATCH_CODE, ReportBody, ReportCard, ReportFindingLine, @@ -52,6 +53,9 @@ import { export type ResolvedUri = { label: string; url: string }; +/** How often a recovery watch polls, and how long it lives. Aggregate conditions floor at 5m. */ +const RECOVERY_WATCH = { checkEveryMinutes: 5, maxHours: 6 } as const; + // --- messages --------------------------------------------------------------- /** @@ -295,6 +299,22 @@ export function ReportView({ const linkByKey = (key: string | undefined) => key === undefined ? undefined : vm.links.find((link) => link.key === key)?.url; + // Only offered when there is something to recover from, and only for the health + // report, which is the one with a recovery watch kind. + const recoveryWatch: AgentIntent | null = + vm.title === "health" && (severity === "warn" || severity === "crit") + ? { + kind: "watch", + spec: { + kind: "health_recovery", + report: "health", + fromSeverity: severity, + note: `${vm.scope} health back to normal`, + ...RECOVERY_WATCH, + }, + } + : null; + // Links a footer action already speaks for aren't repeated as reading matter. const footerLinkKeys = new Set(layout.footer.map((entry) => entry.link).filter(Boolean)); @@ -309,6 +329,22 @@ export function ReportView({ }), })); + if (recoveryWatch && onIntent) { + const watchItem: ReportFooterItem = { + code: FOOTER_WATCH_CODE, + // The label is deliberately the same everywhere; only the pre-filled spec is + // contextual, so a per-object label would break the pattern. + node: onIntent(recoveryWatch)}>Watch…, + }; + // The watch joins the other buttons, before the trailing prose entry. + const noteIndex = footerItems.findIndex((item) => reportFooterStyle(item.code) === "note"); + if (noteIndex !== -1) { + footerItems.splice(noteIndex, 0, watchItem); + } else { + footerItems.push(watchItem); + } + } + // Resources the report cites, resolved to dashboard links by the host. Cited, // not offered, so a text link; our docs still get the docs button. for (const link of vm.links) { diff --git a/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx b/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx new file mode 100644 index 00000000000..0a54ddd894f --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx @@ -0,0 +1,156 @@ +/** + * The banner above a wake narration. + * + * This component holds no kind-specific wording: tone, semantic icon and headline + * come from contracts and `app/presenters/v3/dashboardAgent`. All it decides is which glyph a + * semantic icon draws and which frame a tone paints. + * + * A wake is identified by its message id, `wake:watch:{watchId}:{fired|expired}`. + * That suffix is the transport encoding, not the outcome; the outcome comes off the + * watch row. + */ +import { + CheckCircleIcon, + ClockIcon, + ExclamationCircleIcon, + ExclamationTriangleIcon, + InformationCircleIcon, +} from "@heroicons/react/20/solid"; +import type { + WatchObservedOutcome, + WatchResolution, + WatchSemanticIcon, +} from "@internal/dashboard-agent-contracts"; +import { cn } from "~/utils/cn"; +import { type AgentTone, TONE_ICON_COLOR } from "./agent-badges"; +import { + presentResolvedWatch, + watchSubline, + WATCH_PRESENTATION_FALLBACK, +} from "~/presenters/v3/dashboardAgent"; + +const WAKE_ID_PREFIX = "wake:watch:"; + +/** + * The wire encoding in a wake's message id, not the resolution: `window_completed` + * and `condition_impossible` are both addressed as `expired`, and the row is the + * authority on which one it was. + */ +export type WakeOutcome = "fired" | "expired"; + +/** The watch fields a banner can use. A `WatchChip` satisfies it. */ +export type WakeWatch = { + id: string; + kind: string; + note: string; + identity: string; + /** How the watch ended. Absent on a row written before the resolution model. */ + resolution?: WatchResolution | null; + /** What the resolving check observed — the other half of the headline. */ + observedOutcome?: WatchObservedOutcome | null; + /** + * Why the watch ended, from its last result. Only used to reconstruct a + * resolution for rows that predate the `resolution` column. + */ + endedReason?: string | null; +}; + +export type WakeRef = { watchId: string; outcome: WakeOutcome }; + +/** + * The watch a message narrates the wake of, or null when the message isn't a wake. + * A watch id never ends in an outcome word, so splitting on the last colon is + * unambiguous. + */ +export function wakeRefFromMessageId(messageId: string): WakeRef | null { + if (!messageId.startsWith(WAKE_ID_PREFIX)) return null; + const rest = messageId.slice(WAKE_ID_PREFIX.length); + const split = rest.lastIndexOf(":"); + if (split <= 0) return null; + const outcome = rest.slice(split + 1); + if (outcome !== "fired" && outcome !== "expired") return null; + return { watchId: rest.slice(0, split), outcome }; +} + +/** The watch a wake belongs to, when the host passed its watches down. */ +export function findWakeWatch(watches: WakeWatch[] | undefined, watchId: string) { + return watches?.find((watch) => watch.id === watchId); +} + +/** + * The watch's resolution, falling back to what the transport can prove for a row + * written before the `resolution` column existed: `fired` is unambiguous, `expired` + * splits on the last check's reason. + */ +export function wakeResolution( + outcome: WakeOutcome, + watch: Pick | undefined +): WatchResolution { + if (watch?.resolution) return watch.resolution; + if (outcome === "fired") return "condition_met"; + return watch?.endedReason === "terminal_unsatisfied" + ? "condition_impossible" + : "window_completed"; +} + +/** What this banner shows, without the markup. */ +export function wakePresentation(outcome: WakeOutcome, watch: WakeWatch | undefined) { + if (!watch) return WATCH_PRESENTATION_FALLBACK; + return presentResolvedWatch({ + kind: watch.kind, + identity: watch.identity, + resolution: wakeResolution(outcome, watch), + observed: watch.observedOutcome ?? null, + }); +} + +/** + * Semantic icon to glyph. Which icon a resolved result deserves is decided in + * contracts, and the rule there is that the icon follows the observed outcome, not + * the resolution: a failed run gets `error`, not the check its `condition_met` + * would suggest. + */ +const SEMANTIC_ICON: Record JSX.Element> = { + success: CheckCircleIcon, + attention: ExclamationTriangleIcon, + error: ExclamationCircleIcon, + waiting: ClockIcon, + info: InformationCircleIcon, +}; + +const TONE_FRAME: Record = { + neutral: "border-l-border-bright bg-background-bright/40", + success: "border-l-success bg-success/10", + warning: "border-l-warning bg-warning/10", + error: "border-l-error bg-error/10", +}; + +export function WakeBanner({ + outcome, + watch, +}: { + /** The wire encoding from the wake's message id. */ + outcome: WakeOutcome; + /** The watch that woke, when the host has it. Absent: the neutral fallback. */ + watch?: WakeWatch; +}) { + const presentation = wakePresentation(outcome, watch); + const tone = presentation.tone as AgentTone; + const Icon = SEMANTIC_ICON[presentation.semanticIcon]; + const note = watchSubline(watch); + + return ( +
+ +
+

+ {presentation.label} +

+

{presentation.headline}

+ {note ?

{note}

: null} +
+
+ ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchButton.tsx b/apps/webapp/app/components/dashboard-agent/WatchButton.tsx new file mode 100644 index 00000000000..25b000cd656 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchButton.tsx @@ -0,0 +1,45 @@ +import { EyeIcon } from "@heroicons/react/20/solid"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { Button } from "~/components/primitives/Buttons"; +import { useDashboardAgent } from "./dashboardAgentLauncher"; +import { watchTooltipLabel } from "~/presenters/v3/dashboardAgent"; + +/** Posts nothing: opens the panel with the card pre-filled. Renders nothing without a provider. */ +export function WatchButton({ + spec, + label = "Watch…", + size = "small", + variant = "secondary", + fullWidth, + className, + tooltip, +}: { + spec: WatchSpec; + label?: string; + size?: "small" | "medium"; + variant?: "primary" | "secondary" | "minimal"; + fullWidth?: boolean; + className?: string; + tooltip?: string; +}) { + const agent = useDashboardAgent(); + if (!agent) { + return null; + } + + return ( + + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchCard.tsx b/apps/webapp/app/components/dashboard-agent/WatchCard.tsx new file mode 100644 index 00000000000..e395dcecf10 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchCard.tsx @@ -0,0 +1,330 @@ +/** + * The watch configuration card, opened by the Watch action. + * + * Rules it keeps: the card is ephemeral until submitted (it lives in the panel, + * not the transcript, and only a submitted outcome is persisted as a + * `watch_result` block); Customize expands in place, never a modal; in-chat + * delivery is stated as a line, so the two opt-ins stay independent checkboxes + * and never become a radio group. + * + * Pure component: draft in, markup and callbacks out. Draft rules live in + * `watch-card.ts` and wording in `app/presenters/v3/dashboardAgent`. + */ +import { EyeIcon } from "@heroicons/react/20/solid"; +import { + WATCH_WINDOW_HOURS_OPTIONS, + watchCadenceOptions, + type WatchDraft, + type WatchKind, +} from "@internal/dashboard-agent-contracts"; +import { useId, useState } from "react"; +import { Button } from "~/components/primitives/Buttons"; +import { Checkbox } from "~/components/primitives/Checkbox"; +import { Input } from "~/components/primitives/Input"; +import { AgentSpinner } from "~/components/primitives/Spinner"; +import { cn } from "~/utils/cn"; +import { ChatSystemBlock } from "./chat-layout"; +import { + variantsOf, + watchDraftError, + withAgeMinutes, + withCadence, + withFollowUp, + withThreshold, + withVariant, + withWindow, +} from "./watch-card"; +import { + formatWatchCadence, + formatWatchWindow, + WATCH_IN_CHAT_DELIVERY_LINE, + watchConditionLabel, + watchDurationLabel, + watchSubjectLabel, +} from "~/presenters/v3/dashboardAgent"; + +/** How the condition variants are named in the picker. Short, not sentences. */ +const VARIANT_LABEL: Record = { + run_start: "when it starts", + run_finished: "when it finishes", + run_failed: "if it fails", + backlog_drain: "when it drains", + queue_depth_above: "if it grows", + queue_depth_below: "when it's back below", + queue_stalled: "if it stops moving", + queue_oldest_age: "if runs wait too long", + error_recurrence: "if it recurs", + health_recovery: "when it recovers", +}; + +/** Hoisted so the submit button's icon component keeps a stable identity. */ +function ButtonSpinner() { + return ; +} + +/** Controlled, unlike `CheckboxWithLabel`: the draft is the only thing that says what's on. */ +function Toggle({ + label, + checked, + disabled, + onChange, +}: { + label: string; + checked: boolean; + disabled: boolean; + onChange: (checked: boolean) => void; +}) { + const id = useId(); + return ( +
+ onChange(event.target.checked)} + className="mt-1" + /> + +
+ ); +} + +/** One choice in an inline picker. */ +function Choice({ + selected, + disabled, + onSelect, + children, +}: { + selected: boolean; + disabled: boolean; + onSelect: () => void; + children: React.ReactNode; +}) { + return ( + + ); +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ {label} +
{children}
+
+ ); +} + +export function WatchCard({ + draft, + onChange, + onSubmit, + onCancel, + /** Start expanded: the gallery's Customize state, and a free-text pre-fill. */ + defaultExpanded = false, + /** The submit is in flight: the card stays, disabled, so nothing moves. */ + pending = false, + /** A refusal from the server (cap, duplicate, network). */ + error, +}: { + draft: WatchDraft; + onChange: (draft: WatchDraft) => void; + onSubmit: () => void; + onCancel?: () => void; + defaultExpanded?: boolean; + pending?: boolean; + error?: string | null; +}) { + const [expanded, setExpanded] = useState(defaultExpanded); + const { spec } = draft; + const variants = variantsOf(draft); + // Local validation first: a draft the schema would refuse never reaches the server. + const localError = watchDraftError(draft); + const blocked = localError !== null || pending; + + return ( + } + actions={ + <> + {/* One confirm, expanded or not: an expanded card is submitted as shown. */} + + {!expanded ? ( + + ) : null} + {onCancel ? ( + + ) : null} + + } + > +

+ Watch {watchSubjectLabel(spec)} +

+ {!expanded ? ( + <> +

{watchConditionLabel(spec)}

+

{watchDurationLabel(spec)}

+ + ) : null} +

{WATCH_IN_CHAT_DELIVERY_LINE}

+ + {expanded ? ( +
+ {/* Kinds with no second condition variant must not show an empty picker. */} + {variants.length > 1 ? ( + + {variants.map((kind) => ( + { + if (kind !== spec.kind) onChange(withVariant(draft, kind)); + }} + > + {VARIANT_LABEL[kind]} + + ))} + + ) : ( + + {watchConditionLabel(spec)} + + )} + + {/* One contextual parameter per condition, only where one exists. */} + {spec.kind === "queue_depth_above" || spec.kind === "queue_depth_below" ? ( + + + onChange(withThreshold(draft, Number.parseInt(event.target.value, 10))) + } + aria-label="Queue depth threshold" + /> + + ) : null} + + {spec.kind === "queue_oldest_age" ? ( + + + onChange(withAgeMinutes(draft, Number.parseInt(event.target.value, 10))) + } + aria-label="Wait limit in minutes" + /> + minutes + + ) : null} + + + {WATCH_WINDOW_HOURS_OPTIONS.map((hours) => ( + onChange(withWindow(draft, hours))} + > + {formatWatchWindow(hours)} + + ))} + + + {/* Cadence options come from the kind's schema limits, so an aggregate + watch can never be offered a 1-minute hot loop. */} + + {watchCadenceOptions(spec.kind).map((minutes) => ( + onChange(withCadence(draft, minutes))} + > + {formatWatchCadence(minutes)} + + ))} + + + {/* Two independent opt-ins under a fixed delivery line, never a radio + group, so "email instead of chat" is not expressible. */} + +
+ + onChange(withFollowUp(draft, { investigateOnAttention: checked })) + } + /> + onChange(withFollowUp(draft, { notifyExternally: checked }))} + /> +
+ +
+ ) : null} + + {/* Errors live and die with the card: nothing is persisted. */} + {localError || error ? ( +

{localError ?? error}

+ ) : null} + + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchChips.tsx b/apps/webapp/app/components/dashboard-agent/WatchChips.tsx new file mode 100644 index 00000000000..0fbdce41bab --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchChips.tsx @@ -0,0 +1,113 @@ +import { + CheckCircleIcon, + ClockIcon, + ExclamationCircleIcon, + ExclamationTriangleIcon, + InformationCircleIcon, + NoSymbolIcon, + XMarkIcon, +} from "@heroicons/react/20/solid"; +import type { + WatchObservedOutcome, + WatchResolution, + WatchSemanticIcon, + WatchStatus, +} from "@internal/dashboard-agent-contracts"; +import { AgentSpinner } from "~/components/primitives/Spinner"; +import { SimpleTooltip } from "~/components/primitives/Tooltip"; +import { cn } from "~/utils/cn"; +import { type AgentTone, TONE_ICON_COLOR } from "./agent-badges"; +import { wakePresentation } from "./WakeBanner"; +import { watchChipLabel, watchChipTooltip } from "./watch-chips"; + +/** As the panel's loader hands it over: dates are already JSON strings. */ +export type WatchChip = { + id: string; + identity: string; + status: WatchStatus; + kind: string; + note: string; + checkEveryMinutes: number; + expiresAt: string; + endedReason?: string | null; + /** Null while active; absent on rows written before the resolution column. */ + resolution?: WatchResolution | null; + observedOutcome?: WatchObservedOutcome | null; +}; + +const SEMANTIC_ICON: Record JSX.Element> = { + success: CheckCircleIcon, + attention: ExclamationTriangleIcon, + error: ExclamationCircleIcon, + waiting: ClockIcon, + info: InformationCircleIcon, +}; + +/** + * A terminal chip wears the resolved result's icon, not its lifecycle status: a + * `run_finished` watch on a failed run resolves `condition_met`. Cancellation has none. + */ +function StatusIcon({ watch }: { watch: WatchChip }) { + if (watch.status === "active") return ; + + if (watch.status === "cancelled") { + return ; + } + + const presentation = wakePresentation(watch.status === "fired" ? "fired" : "expired", watch); + const Icon = SEMANTIC_ICON[presentation.semanticIcon]; + return ( + + ); +} + +export function WatchChips({ + watches, + onCancel, +}: { + watches: WatchChip[]; + onCancel?: (watchId: string) => void; +}) { + if (watches.length === 0) return null; + + return ( +
+ watches + {watches.map((watch) => { + const label = watchChipLabel(watch); + return ( + + + {label}} + /> + {watch.status === "active" && onCancel ? ( + onCancel(watch.id)} + className="text-text-faint transition-colors hover:text-error focus-visible:text-error focus-custom" + > + + + } + /> + ) : null} + + ); + })} +
+ ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx b/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx new file mode 100644 index 00000000000..551a9683ef5 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx @@ -0,0 +1,48 @@ +/** + * What a submitted watch card leaves in the transcript, in two flavours. + * + * A confirmation states the watch's lifetime facts and is the only transcript record + * of the request. A one-shot result means the immediate check answered outright and + * no watch was created, so no chip appears, no wake arrives and there is nothing to + * cancel. + * + * Pure component: the wording is not computed here, it was frozen into the block at + * append time by `app/presenters/v3/dashboardAgent`, so a later copy change never rewrites what + * a user was already told. + */ +import { CheckCircleIcon, EyeIcon, InformationCircleIcon } from "@heroicons/react/20/solid"; +import type { WatchResultBlock as WatchResultBlockPayload } from "@internal/dashboard-agent-contracts"; +import { ChatSystemBlock } from "./chat-layout"; +import { TONE_ICON_COLOR } from "./agent-badges"; +import { cn } from "~/utils/cn"; + +/** + * Icon and label per outcome. A confirmation is not a success (nothing has happened + * yet) so it wears the neutral eye; the check belongs to the one-shot that did + * answer the question. + */ +const OUTCOME = { + watching: { label: "Watch", Icon: EyeIcon, tone: "neutral" }, + already_true: { label: "Watch", Icon: CheckCircleIcon, tone: "success" }, + impossible: { label: "Watch", Icon: InformationCircleIcon, tone: "neutral" }, +} as const; + +export function WatchResultBlock({ block }: { block: WatchResultBlockPayload }) { + const { label, Icon, tone } = OUTCOME[block.outcome] ?? OUTCOME.watching; + + return ( + } + > +

{block.headline}

+ {block.lifetime ?

{block.lifetime}

: null} + {block.detail ?

{block.detail}

: null} + {(block.followUp ?? []).map((line) => ( +

+ {line} +

+ ))} + + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx b/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx new file mode 100644 index 00000000000..8170c37b07f --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx @@ -0,0 +1,133 @@ +/** + * The dashboard-wide signal that a watch woke a chat while the panel was closed. + * + * Persistent by design: a wake answers a question asked minutes or hours ago, so it + * waits until dismissed rather than expiring on a timer. Dismissing does not mark + * the chat read (reading happens in the panel), so the launcher's dot survives a + * swatted toast. + */ +import { toast } from "sonner"; +import { Button } from "~/components/primitives/Buttons"; +import { ToastUI } from "~/components/primitives/Toast"; +import type { WatchObservedOutcome, WatchResolution } from "@internal/dashboard-agent-contracts"; +import { wakeResolution } from "./WakeBanner"; +import { presentResolvedWatch, WATCH_PRESENTATION_FALLBACK } from "~/presenters/v3/dashboardAgent"; + +/** Matches sonner's default toast width, same as the app's other toasts. */ +const TOAST_WIDTH = 356; + +/** More new wakes than this at once collapse into one summary toast. */ +export const WAKE_TOAST_MAX_INDIVIDUAL = 3; + +export type WatchWake = { + watchId: string; + chatId: string; + /** The wire encoding off the row. Not the outcome; see `resolution`. */ + outcome: "fired" | "expired"; + note: string; + /** + * What actually happened, frozen on the row by the resolving check. The toast, + * the banner and the email take their headline from the same presenter so they + * cannot disagree. Absent on a row written before the resolution model, where the + * presenter falls back rather than guessing. + */ + kind?: string; + identity?: string; + resolution?: WatchResolution | null; + observedOutcome?: WatchObservedOutcome | null; + /** Landed after the chat's read marker. The dot counts these; the toast fires either way. */ + unread?: boolean; +}; + +/** + * The toast's title: the fact, or the neutral fallback when this wake predates the + * resolution model. The wording is `app/presenters/v3/dashboardAgent`'s; this only decides + * which watch to ask it about. + */ +export function watchWakeToastTitle(wake: WatchWake): string { + if (!wake.kind || !wake.identity) return WATCH_PRESENTATION_FALLBACK.headline; + return presentResolvedWatch({ + kind: wake.kind, + identity: wake.identity, + resolution: wakeResolution(wake.outcome, { resolution: wake.resolution ?? null }), + observed: wake.observedOutcome ?? null, + }).headline; +} + +function WakeToastUI({ + t, + title, + message, + onOpenChat, +}: { + t: string; + title: string; + message: string; + onOpenChat: () => void; +}) { + return ( + { + onOpenChat(); + toast.dismiss(t); + }} + > + Open chat + + } + /> + ); +} + +function show(node: (t: string) => React.ReactElement, id: string) { + toast.custom((t) => node(t as string), { + // Manual dismissal only — see the file comment. + duration: Infinity, + // Keyed so a re-render or a duplicate poll can't stack the same wake twice. + id, + }); +} + +/** + * One persistent toast for a single wake. `onOpenChat` is given the chat the wake + * happened in, not whichever chat the panel had open last. + */ +export function showWatchWakeToast(wake: WatchWake, onOpenChat: (chatId: string) => void) { + show( + (t) => ( + onOpenChat(wake.chatId)} + /> + ), + `watch-wake-${wake.watchId}` + ); +} + +/** One persistent toast standing in for a batch too large to narrate one by one. */ +export function showWatchWakesSummaryToast(count: number, onOpenChat: () => void) { + show( + (t) => ( + + ), + // One id for all summaries: a later poll rewrites the count in place instead + // of stacking a second never-expiring toast on top of the first. + "watch-wakes-summary" + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts b/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts index 12a8cc8ff85..f58408f500d 100644 --- a/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts +++ b/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts @@ -80,6 +80,7 @@ describe("chat-layout enforcement", () => { "ChatToolRow", "ChatNote", "ChatStatusLine", + "ChatWakeSlot", "ChatActionsRow", ]) { expect(source, name).toContain(`export function ${name}(`); diff --git a/apps/webapp/app/components/dashboard-agent/chat-layout.tsx b/apps/webapp/app/components/dashboard-agent/chat-layout.tsx index 7139f24b07b..d281f64b6a5 100644 --- a/apps/webapp/app/components/dashboard-agent/chat-layout.tsx +++ b/apps/webapp/app/components/dashboard-agent/chat-layout.tsx @@ -12,6 +12,7 @@ const TURN_GAP = "space-y-4"; const TURN_BODY_GAP = "space-y-2"; const ROW_GAP = "gap-2"; const CHIP_GAP = "gap-1.5"; +const UNIT_GAP = "space-y-1.5"; const SCROLLER = "flex-1 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control"; @@ -148,6 +149,21 @@ export function ChatStatusLine({ ); } +export function ChatWakeSlot({ + banner, + children, +}: { + banner: React.ReactNode; + children: React.ReactNode; +}) { + return ( +
+ {banner} + {children} +
+ ); +} + const BLOCK_LINE_GAP = "space-y-1"; const BLOCK_INSET = "px-3 py-2.5"; diff --git a/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx b/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx index f602e89a533..6104e8a000a 100644 --- a/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx +++ b/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx @@ -1,3 +1,4 @@ +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; import { createContext, useContext } from "react"; import { Button } from "~/components/primitives/Buttons"; import { ShortcutKey } from "~/components/primitives/ShortcutKey"; @@ -26,6 +27,12 @@ type DashboardAgentContextValue = { setOpen: (open: boolean) => void; /** Sent as the first message of a new chat; with a chat open it only fills the composer. */ openWith: (text: string) => void; + /** Nothing is posted or persisted until the card is submitted. */ + openWithWatch: (spec: WatchSpec) => void; + /** Polled only while the panel is closed; 0 while it is open. */ + unreadWakes: number; + /** Chats that answered, settled or woke while the panel was closed. */ + unreadWork: number; }; const DashboardAgentContext = createContext(null); @@ -43,11 +50,13 @@ export function DashboardAgentLauncher() { return null; } - const { open, setOpen } = agent; + const { open, setOpen, unreadWakes, unreadWork } = agent; if (open) { return null; } + const hasUnread = unreadWakes > 0 || unreadWork > 0; + return ( + {hasUnread && ( + // The ring matches the `NavBar` surface the launcher sits on. + + )} } /> diff --git a/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts b/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts index 3fd4699146d..2cae37107b9 100644 --- a/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts +++ b/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts @@ -31,7 +31,7 @@ export const demoConcurrencySaturationSignal: AgentPageSignal = { severity: "crit", }; -// Priority order. +// Priority order. `SIGNAL_PRIORITY` in the registry mirrors this. export const demoSignalsByPriority: AgentPageSignal[] = [ demoFreshFailureSignal, demoWaitingRunSignal, @@ -141,6 +141,12 @@ export const demoPromptSets: Record = { "How many other runs failed with this error in the last hour?", "contextual" ), + prompt( + "watch-retry", + "Tell me when it retries", + `Watch ${DEMO_WORLD.failedRunId} and tell me when it finishes.`, + "contextual" + ), DEFAULT_PROMPTS[1]!, ], waitingRun: [ @@ -195,6 +201,12 @@ export const demoPromptSets: Record = { "Explain this error and what usually causes it.", "promoted" ), + prompt( + "watch-recurrence", + "Tell me if it comes back", + "Watch this error and tell me if it happens again.", + "contextual" + ), DEFAULT_PROMPTS[1]!, ], queue: [ @@ -224,7 +236,7 @@ export const demoPromptSets: Record = { other: DEFAULT_PROMPTS, }; -export const demoDismissedPromptIds: string[] = []; +export const demoDismissedPromptIds: string[] = [demoId("prompt-watch-retry")]; export const demoPromptsAfterDismissal: SuggestedPrompt[] = demoPromptSets.failedRun .filter((p) => !demoDismissedPromptIds.includes(p.id)) diff --git a/apps/webapp/app/components/dashboard-agent/list-row.tsx b/apps/webapp/app/components/dashboard-agent/list-row.tsx index 6fe35f267e3..a5b80d873ff 100644 --- a/apps/webapp/app/components/dashboard-agent/list-row.tsx +++ b/apps/webapp/app/components/dashboard-agent/list-row.tsx @@ -20,6 +20,7 @@ export function AgentListRow({ meta, status, variant = "default", + unread = false, onSelect, action, }: { @@ -27,6 +28,7 @@ export function AgentListRow({ meta?: ReactNode; status?: ReactNode; variant?: AgentListRowVariant; + unread?: boolean; onSelect: () => void; /** Use {@link AgentListRowAction}. */ action?: ReactNode; @@ -38,12 +40,19 @@ export function AgentListRow({ onClick={onSelect} className={cn( "flex min-w-0 flex-1 items-center gap-2 rounded-md border px-3 py-2 text-left text-sm outline-hidden transition focus-custom", - ROW_VARIANTS[variant] + ROW_VARIANTS[variant], + unread && "text-text-bright" )} > {status ? ( {status} ) : null} + {unread ? ( + <> + + Unread. + + ) : null} {label} {meta ? {meta} : null} diff --git a/apps/webapp/app/components/dashboard-agent/message-quota.ts b/apps/webapp/app/components/dashboard-agent/message-quota.ts index 21edb9ab9fb..f65481c8705 100644 --- a/apps/webapp/app/components/dashboard-agent/message-quota.ts +++ b/apps/webapp/app/components/dashboard-agent/message-quota.ts @@ -1,3 +1,5 @@ +import { isWatchRequestMessageId } from "@internal/dashboard-agent-contracts"; + // Counted per user across their chats in the org, not per chat, which "New chat" // would reset. export const FREE_PLAN_MESSAGE_LIMIT = 20; @@ -25,6 +27,12 @@ export function resolveMessageQuota({ : { kind: "within", used, limit, remaining }; } -export function countUserMessages(messages: { role: string }[]): number { - return messages.reduce((total, message) => (message.role === "user" ? total + 1 : total), 0); +// A watch's consent record is a user message the person never typed, so it is +// excluded here exactly as the stored count excludes it. +export function countUserMessages(messages: { role: string; id?: string }[]): number { + return messages.reduce( + (total, message) => + message.role === "user" && !isWatchRequestMessageId(message.id) ? total + 1 : total, + 0 + ); } diff --git a/apps/webapp/app/components/dashboard-agent/panel-escape.test.ts b/apps/webapp/app/components/dashboard-agent/panel-escape.test.ts new file mode 100644 index 00000000000..cfb57828394 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/panel-escape.test.ts @@ -0,0 +1,75 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { escapeClosesPanel } from "./panel-escape"; + +/** + * Escape has to reach the thing the user meant. Radix dismisses a popover or a dialog from a + * document listener that runs after the panel's own handler and never marks the event handled, + * so the panel has to decide for itself whether the keystroke came from inside it. + */ +describe("escapeClosesPanel", () => { + it("closes the panel when Escape comes from the panel itself", () => { + expect( + escapeClosesPanel({ key: "Escape", defaultPrevented: false, targetInsidePanel: true }) + ).toBe(true); + }); + + it("leaves the panel open when Escape comes from a portalled layer", () => { + // The history popover and the delete dialog both render outside the panel's DOM subtree. + expect( + escapeClosesPanel({ key: "Escape", defaultPrevented: false, targetInsidePanel: false }) + ).toBe(false); + }); + + it("stays out of the way once something else has handled the key", () => { + expect( + escapeClosesPanel({ key: "Escape", defaultPrevented: true, targetInsidePanel: true }) + ).toBe(false); + }); + + it("ignores every other key", () => { + expect( + escapeClosesPanel({ key: "Enter", defaultPrevented: false, targetInsidePanel: true }) + ).toBe(false); + expect(escapeClosesPanel({ key: "j", defaultPrevented: false, targetInsidePanel: true })).toBe( + false + ); + }); +}); + +/** + * Structural guards, not behavioural proof: the delete confirmation's survival depends on where + * it is mounted in the tree, which these assertions pin down without rendering anything. + */ +describe("the delete confirmation lives outside the history popover", () => { + const header = readFileSync(new URL("./DashboardAgentHeader.tsx", import.meta.url), "utf8"); + const history = readFileSync(new URL("./DashboardAgentHistory.tsx", import.meta.url), "utf8"); + const panel = readFileSync(new URL("./DashboardAgentPanel.tsx", import.meta.url), "utf8"); + + const menuBody = history.slice( + history.indexOf("export function DashboardAgentHistoryMenu"), + history.indexOf("export function DashboardAgentDeleteChatDialog") + ); + + it("keeps no dialog and no pending state inside the popover's menu", () => { + expect(menuBody).not.toContain(" { + const popoverEnd = header.indexOf(""); + const dialog = header.indexOf(" { + expect(header).toContain("const [pendingDelete, setPendingDelete] = useState"); + }); + + it("gates the panel's Escape on the shared rule rather than defaultPrevented alone", () => { + expect(panel).toContain("escapeClosesPanel({"); + expect(panel).toContain("panelRef.current?.contains(event.target as Node)"); + expect(panel).not.toContain('if (event.key !== "Escape" || event.defaultPrevented) return;'); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/panel-escape.ts b/apps/webapp/app/components/dashboard-agent/panel-escape.ts new file mode 100644 index 00000000000..201d7b09c18 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/panel-escape.ts @@ -0,0 +1,15 @@ +/** + * Escape inside the panel closes the panel — but a popover or a dialog is portalled out of + * the panel's DOM subtree while still bubbling through the React tree, and Radix dismisses + * those from a document listener that runs after this handler, so the event arrives here + * undefaulted. Deciding on the DOM target is what tells the two apart. + */ +export function escapeClosesPanel(event: { + key: string; + defaultPrevented: boolean; + /** Whether the event's target is a DOM descendant of the panel. */ + targetInsidePanel: boolean; +}): boolean { + if (event.key !== "Escape" || event.defaultPrevented) return false; + return event.targetInsidePanel; +} diff --git a/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts b/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts index 6cbc02d3573..2ff0d61403f 100644 --- a/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts +++ b/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { pendingNavigateIntents } from "./pending-intents"; +import { pendingNavigateIntents, pendingWatchIntents } from "./pending-intents"; describe("pendingNavigateIntents", () => { const uri = "trigger://proj_abc/env_123/run/run_abc"; @@ -46,3 +46,75 @@ describe("pendingNavigateIntents", () => { ).toEqual([{ kind: "navigate", target: uri }]); }); }); + +describe("pendingWatchIntents", () => { + const spec = { + kind: "run_finished", + runId: "run_abc", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when the receipt run finishes", + }; + const toolPart = (toolCallId: string, state = "output-available") => ({ + type: "tool-schedule_watch", + state, + toolCallId, + output: { intent: { kind: "watch", spec } }, + }); + + it("returns the proposed spec from a completed schedule_watch call, once", () => { + const seen = new Set(); + const messages = [{ id: "m1", parts: [toolPart("call-1")] }]; + + expect(pendingWatchIntents(messages, seen)).toEqual([{ kind: "watch", spec }]); + expect(pendingWatchIntents(messages, seen)).toEqual([]); + }); + + it("ignores a call still running, and a spec the contract rejects", () => { + expect( + pendingWatchIntents([{ id: "m1", parts: [toolPart("call-1", "input-available")] }], new Set()) + ).toEqual([]); + + const invalid = [ + { + id: "m1", + parts: [ + { + ...toolPart("call-2"), + output: { intent: { kind: "watch", spec: { kind: "run_finished" } } }, + }, + ], + }, + ]; + expect(pendingWatchIntents(invalid, new Set())).toEqual([]); + }); + + it("never reopens a proposal seeded from loaded history", () => { + const history = [{ id: "m1", parts: [toolPart("call-1")] }]; + const seen = new Set(); + pendingWatchIntents(history, seen); + + expect(pendingWatchIntents(history, seen)).toEqual([]); + expect( + pendingWatchIntents([...history, { id: "m2", parts: [toolPart("call-2")] }], seen) + ).toEqual([{ kind: "watch", spec }]); + }); + + it("doesn't confuse a navigate result for a watch", () => { + const messages = [ + { + id: "m1", + parts: [ + { + type: "tool-navigate_to", + state: "output-available", + toolCallId: "call-1", + output: { intent: { kind: "navigate", target: "trigger://p/e/run/run_abc" } }, + }, + ], + }, + ]; + + expect(pendingWatchIntents(messages, new Set())).toEqual([]); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/pending-intents.ts b/apps/webapp/app/components/dashboard-agent/pending-intents.ts index 6b58ba04d31..93a1d790656 100644 --- a/apps/webapp/app/components/dashboard-agent/pending-intents.ts +++ b/apps/webapp/app/components/dashboard-agent/pending-intents.ts @@ -40,3 +40,11 @@ export function pendingNavigateIntents( ): Array> { return pendingToolIntents(messages, seen, "tool-navigate_to", "navigate"); } + +// `schedule_watch` only proposes: the panel creates the watch. +export function pendingWatchIntents( + messages: ReadonlyArray, + seen: Set +): Array> { + return pendingToolIntents(messages, seen, "tool-schedule_watch", "watch"); +} diff --git a/apps/webapp/app/components/dashboard-agent/pending-turn.test.ts b/apps/webapp/app/components/dashboard-agent/pending-turn.test.ts new file mode 100644 index 00000000000..0dee765fb80 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/pending-turn.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { nextPendingTurnChatId } from "./pending-turn"; +import { shouldPollWakeFeed } from "./watch-activity"; + +/** + * The launcher dot only appears if the wake poll is running when the answer lands. A turn + * started in the panel has to keep the poll alive across a close, and let go of it once the + * answer has been seen. + */ +describe("nextPendingTurnChatId", () => { + it("latches onto the chat whose turn started", () => { + expect(nextPendingTurnChatId(null, { chatId: "chat_a", active: true })).toBe("chat_a"); + }); + + it("holds while a newer turn takes over", () => { + const afterA = nextPendingTurnChatId(null, { chatId: "chat_a", active: true }); + expect(nextPendingTurnChatId(afterA, { chatId: "chat_b", active: true })).toBe("chat_b"); + }); + + it("lets go once that chat's turn is no longer running", () => { + const pending = nextPendingTurnChatId(null, { chatId: "chat_a", active: true }); + expect(nextPendingTurnChatId(pending, { chatId: "chat_a", active: false })).toBe(null); + }); + + it("keeps waiting when a different chat goes quiet", () => { + const pending = nextPendingTurnChatId(null, { chatId: "chat_a", active: true }); + expect(nextPendingTurnChatId(pending, { chatId: "chat_b", active: false })).toBe("chat_a"); + }); + + it("stays clear when nothing is pending", () => { + expect(nextPendingTurnChatId(null, { chatId: "chat_a", active: false })).toBe(null); + }); +}); + +describe("a turn started behind a closed panel", () => { + // The page load knew of nothing: no wake, no watch, no unread work. Only the turn can + // start the poll. + const quietPageLoad = { + serverUnreadWakes: 0, + serverHasActiveWatches: false, + serverUnreadWork: 0, + organizationId: "org_quiet", + }; + + it("keeps the poll running until the answer is seen", () => { + expect(shouldPollWakeFeed({ ...quietPageLoad, turnInFlight: false })).toBe(false); + + // Asked a question, then closed the panel: the panel reports no end, so the latch holds. + const pending = nextPendingTurnChatId(null, { chatId: "chat_a", active: true }); + expect(shouldPollWakeFeed({ ...quietPageLoad, turnInFlight: pending !== null })).toBe(true); + + // Re-opened the chat with the turn already over. + const seen = nextPendingTurnChatId(pending, { chatId: "chat_a", active: false }); + expect(shouldPollWakeFeed({ ...quietPageLoad, turnInFlight: seen !== null })).toBe(false); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/pending-turn.ts b/apps/webapp/app/components/dashboard-agent/pending-turn.ts new file mode 100644 index 00000000000..d024d19ea59 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/pending-turn.ts @@ -0,0 +1,17 @@ +/** + * Which chat this tab is still waiting on. A turn started here can finish after the panel + * closes — the case the launcher dot exists for — so the wake poll has to keep running until + * the answer has been seen. The panel reports turn activity while it is mounted; closing it + * reports nothing, which is what leaves the latch set. + */ + +/** `active` is true while a turn is running in `chatId`, false once it is not. */ +export function nextPendingTurnChatId( + current: string | null, + event: { chatId: string; active: boolean } +): string | null { + if (event.active) return event.chatId; + // Only the chat we are waiting on clears the latch; another chat going quiet says nothing + // about this one. + return current === event.chatId ? null : current; +} diff --git a/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx b/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx index c4c8f248133..add6fb256ff 100644 --- a/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx +++ b/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx @@ -310,6 +310,13 @@ export function ReportNoteBlock({ label, children }: { label: string; children: // and `note` is prose for an option stated rather than offered. export { reportFooterStyle, type ReportFooterStyle }; +/** + * The recovery-watch offer. No report emits it; the card adds it. Two codes + * because it is phrased differently when it is the only thing on offer. + */ +export const FOOTER_WATCH_CODE = "watch_recovery"; +export const FOOTER_WATCH_ONLY_CODE = "watch_recovery_only"; + /** A dimmed line that accompanies a row entry. */ const FOOTER_NOTE_LINES: Record = { check_control_plane: "There's nothing to fix on your side.", diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts index 9ea3216bb91..4be9d9acc8f 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts @@ -206,7 +206,7 @@ describe("queueAgentPageContext", () => { const context = queueAgentPageContext(queueLoaderData()); expect(context).toEqual({ - page: { kind: "queue", name: "black-friday", health: "ok" }, + page: { kind: "queue", name: "black-friday", health: "ok", paused: false }, signals: [], }); expect(agentPageContextSchema.safeParse(context).success).toBe(true); @@ -244,6 +244,16 @@ describe("queueAgentPageContext", () => { expect(context?.signals).toEqual([]); }); + it("offers no watch on a paused queue, even when it is at capacity", () => { + // Paused and saturated at once: nothing will drain or grow until it is resumed, so a + // watch would promise an answer that can't come. + const context = queueAgentPageContext( + queueLoaderData({ paused: true, running: 10, queued: 40, concurrencyLimit: 10 }) + ); + + expect(context?.signals).toEqual([]); + }); + it("emits nothing for an unlimited queue, however deep the backlog", () => { const context = queueAgentPageContext( queueLoaderData({ concurrencyLimit: null, running: 99, queued: 99 }) diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.ts index 142ad9c0050..b18c4bfd162 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.ts @@ -210,12 +210,13 @@ export function queueAgentPageContext(data: unknown): AgentPageContext | undefin const health = atCapacity ? "crit" : paused || queued > 0 || waitingTooLong ? "warn" : "ok"; const signals: AgentPageSignal[] = []; - if (atCapacity) { + // Nothing to watch on a paused queue: it can neither drain nor grow until it is resumed. + if (atCapacity && !paused) { // A backlog at least as deep as the limit won't clear this cycle. signals.push({ kind: "concurrency_saturation", severity: queued >= limit! ? "crit" : "warn" }); } - return { page: { kind: "queue", name, health }, signals }; + return { page: { kind: "queue", name, health, paused: Boolean(paused) }, signals }; } export function deploymentsAgentPageContext(): AgentPageContext { diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts index 8f594a1183c..1bc7aeb340d 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts @@ -97,6 +97,11 @@ export function pageSlotPrompts(page: AgentPage): PageSlotPrompts { "Why does this keep happening?", "Investigate this error — why does it keep coming back, and which runs are affected?" ), + watch: def( + "error-watch-recurrence", + "Tell me if it comes back", + "Watch this error and tell me if it happens again." + ), explain: def( "error-similar", "Find similar failures", @@ -117,14 +122,23 @@ export function pageSlotPrompts(page: AgentPage): PageSlotPrompts { case "queue": return { + // A paused queue is backed up because someone paused it, and nothing it could be + // watched for will happen until they resume it — so neither chip is offered. investigate: - page.health === "warn" || page.health === "crit" + !page.paused && (page.health === "warn" || page.health === "crit") ? def( "queue-backlog-cause", "Why is this queue backed up?", queueBacklogPrompt(page.name) ) : undefined, + watch: page.paused + ? undefined + : def( + "queue-watch-drain", + "Tell me when the backlog drains", + `Watch the ${page.name} queue and tell me when the backlog drains.` + ), status: def( "queue-backlog", "How big is the backlog?", diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts index aae8e0e4cf0..5e5f7e5690a 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts @@ -25,12 +25,13 @@ export const ctx = (id: string, label: string, prompt: string) => make(id, label, prompt, "contextual"); /** The slots after the promoted one, in display order. */ -export const PROMPT_SLOTS = ["investigate", "status", "explain", "docs"] as const; +export const PROMPT_SLOTS = ["investigate", "watch", "status", "explain", "docs"] as const; export type PromptSlot = (typeof PROMPT_SLOTS)[number]; export type PageSlotPrompts = { investigate?: SuggestedPrompt; + watch?: SuggestedPrompt; status?: SuggestedPrompt; explain: SuggestedPrompt; docs: SuggestedPrompt; diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts index 80ce032b0bf..97f4ec558d2 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts @@ -25,27 +25,33 @@ const docsId = (key: keyof typeof demoPageContexts) => pageSlotPrompts(demoPageContexts[key].page).docs.id; describe("resolveSuggestedPrompts", () => { - it("fills every slot the page has, given a promoted chip, signals and defaults", () => { + it("fills all five slots when the page has a promoted chip, signals and defaults", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.error, { promoted, now: NOW }); - expect(prompts).toHaveLength(4); + expect(prompts).toHaveLength(5); expect(ids(prompts)).toEqual([ promoted.id, "sp:fresh-failure", + "sp:error-watch-recurrence", "sp:error-similar", docsId("error"), ]); expect(prompts[0]?.source).toBe("promoted"); }); - it("drops a slot when nothing is promoted", () => { + it("drops to four slots when nothing is promoted", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW }); - expect(prompts).toHaveLength(3); - expect(ids(prompts)).toEqual(["sp:fresh-failure", "sp:error-similar", docsId("error")]); + expect(prompts).toHaveLength(4); + expect(ids(prompts)).toEqual([ + "sp:fresh-failure", + "sp:error-watch-recurrence", + "sp:error-similar", + docsId("error"), + ]); }); - it("shows explain + docs only when no investigate applies", () => { + it("shows explain + docs only when no investigate or watch applies", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.deployment, { now: NOW }); expect(ids(prompts)).toEqual(ids(pageDefaultPrompts(demoPageContexts.deployment.page))); @@ -73,7 +79,7 @@ describe("resolveSuggestedPrompts", () => { } }); - it("orders promoted, then investigate, then status, then explain", () => { + it("orders promoted, then investigate, then watch, then explain", () => { const context = { ...demoPageContexts.queue, signals: [...demoPageContexts.queue.signals, demoFreshFailureSignal], @@ -84,7 +90,8 @@ describe("resolveSuggestedPrompts", () => { expect(ids(prompts)).toEqual([ promoted.id, "sp:fresh-failure", - "sp:queue-backlog", + // waiting_run beats concurrency_saturation for the watch slot. + "sp:waiting-run", "sp:queue-state", docsId("queue"), ]); @@ -122,10 +129,10 @@ describe("resolveSuggestedPrompts", () => { const full = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW }); const dismissed = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW, - dismissedIds: ["sp:error-similar"], + dismissedIds: ["sp:error-watch-recurrence"], }); - expect(ids(dismissed)).not.toContain("sp:error-similar"); + expect(ids(dismissed)).not.toContain("sp:error-watch-recurrence"); expect(dismissed).toHaveLength(full.length - 1); expect(dismissed.at(-1)?.id).toBe(docsId("error")); }); @@ -161,7 +168,12 @@ describe("resolveSuggestedPrompts", () => { expect(failure?.prompt).toContain("12m ago"); }); - it("words the slow-run chip for its slot", () => { + it("words the waiting-run and slow-run chips for their slots", () => { + const waiting = resolveSuggestedPrompts(demoPageContexts.waitingRun, { now: NOW }); + const waitingChip = waiting.find((p) => p.id === "sp:waiting-run"); + expect(waitingChip?.label).toBe("Tell me when this run starts"); + expect(waitingChip?.prompt).toContain("queue"); + const slow = resolveSuggestedPrompts(demoPageContexts.slowRun, { now: NOW }); expect(slow[0]?.label).toBe("~7.8x slower than usual"); }); @@ -223,6 +235,17 @@ describe("pageSlotPrompts", () => { } }); + it("offers a paused queue neither chip, however unhealthy it looks", () => { + // Paused reads as `warn`, so without the guard the backlog chips would both appear — + // asking why a queue someone paused is backed up, and offering to watch it drain. + const slots = pageSlotPrompts({ kind: "queue", name: "emails", health: "warn", paused: true }); + + expect(slots.investigate).toBeUndefined(); + expect(slots.watch).toBeUndefined(); + // The page is still explainable; only the two backlog asks are withheld. + expect(slots.explain).toBeDefined(); + }); + it("offers a deployment investigate chip only for a deploy that didn't land", () => { expect(pageSlotPrompts({ kind: "deployment", version: "1.0" }).investigate).toBeUndefined(); expect( diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts index 9264ae78690..b8575acd459 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts @@ -68,7 +68,7 @@ export function resolveSuggestedPromptsBySlot( } // Over the cap, optional slots yield in this order; promoted, explain and docs never yield. - const yieldOrder: ResolvedPromptSlot[] = ["status", "investigate"]; + const yieldOrder: ResolvedPromptSlot[] = ["status", "watch", "investigate"]; let trimmed = resolved; for (const slot of yieldOrder) { if (trimmed.length <= SUGGESTED_PROMPT_CAP) break; diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts index d5faf17653f..6c4a2f74982 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts @@ -10,14 +10,20 @@ import type { } from "@internal/dashboard-agent-contracts"; import { ctx, type PromptSlot } from "./prompt-chips"; -/** A kind with no entry produces no chip. */ -export const SIGNAL_SLOT: Partial> = { +export const SIGNAL_SLOT: Record = { fresh_failure: "investigate", slow_run: "investigate", + waiting_run: "watch", + concurrency_saturation: "watch", }; -/** Signal precedence within a slot. */ -export const SIGNAL_PRIORITY: AgentPageSignalKind[] = ["fresh_failure", "slow_run"]; +/** Signal precedence within a slot. Mirrors `demoSignalsByPriority` in the fixtures. */ +export const SIGNAL_PRIORITY: AgentPageSignalKind[] = [ + "fresh_failure", + "waiting_run", + "slow_run", + "concurrency_saturation", +]; /** "3m", "2h", "4d". */ export function formatAgo(ms: number): string { @@ -47,6 +53,15 @@ export function promptForSignal(signal: AgentPageSignal, now: number): Suggested ); } + case "waiting_run": + return ctx( + "waiting-run", + "Tell me when this run starts", + signal.queue + ? `Watch ${signal.runId} and tell me when it leaves the ${signal.queue} queue.` + : `Watch ${signal.runId} and tell me when it starts running.` + ); + case "slow_run": { if (signal.baselineP95Ms <= 0) return undefined; const factor = formatMultiplier(signal.durationMs / signal.baselineP95Ms); @@ -56,6 +71,13 @@ export function promptForSignal(signal: AgentPageSignal, now: number): Suggested `${signal.runId} is running ~${factor} slower than this task's usual p95. Investigate why.` ); } + + case "concurrency_saturation": + return ctx( + "concurrency-saturation", + "Tell me when the backlog drains", + "Concurrency is saturated right now. Watch it and tell me when the backlog drains." + ); } } @@ -79,18 +101,17 @@ export function contextualPromptsBySlot( ): Record { const bySlot: Record = { investigate: [], + watch: [], status: [], explain: [], docs: [], }; for (const kind of SIGNAL_PRIORITY) { - const slot = SIGNAL_SLOT[kind]; - if (!slot) continue; for (const signal of context.signals) { if (signal.kind !== kind) continue; const prompt = promptForSignal(signal, now); - if (prompt) bySlot[slot].push(prompt); + if (prompt) bySlot[SIGNAL_SLOT[kind]].push(prompt); } } diff --git a/apps/webapp/app/components/dashboard-agent/tool-labels.ts b/apps/webapp/app/components/dashboard-agent/tool-labels.ts index d3bd07f52d4..4972cdb2d5b 100644 --- a/apps/webapp/app/components/dashboard-agent/tool-labels.ts +++ b/apps/webapp/app/components/dashboard-agent/tool-labels.ts @@ -21,6 +21,7 @@ const TOOL_LABELS: Record = { search_docs: "Searching the docs", get_current_page: "Reading the current page", navigate_to: "Opening the page", + schedule_watch: "Filling in a watch", list_alerts: "Listing alerts", create_alert: "Creating an alert", delete_alert: "Deleting an alert", diff --git a/apps/webapp/app/components/dashboard-agent/turn-error.test.ts b/apps/webapp/app/components/dashboard-agent/turn-error.test.ts index 9975bdd1f49..4bf646e4fbb 100644 --- a/apps/webapp/app/components/dashboard-agent/turn-error.test.ts +++ b/apps/webapp/app/components/dashboard-agent/turn-error.test.ts @@ -7,7 +7,7 @@ const failure = { id: "turn-error:0" }; describe("the failed-turn record", () => { it("recognises the agent's failure message id", () => { expect(isTurnErrorMessageId("turn-error:3")).toBe(true); - expect(isTurnErrorMessageId("msg_1")).toBe(false); + expect(isTurnErrorMessageId("wake:watch:watch_1:fired")).toBe(false); expect(isTurnErrorMessageId(undefined)).toBe(false); }); diff --git a/apps/webapp/app/components/dashboard-agent/turn-error.ts b/apps/webapp/app/components/dashboard-agent/turn-error.ts index 1fecf35c1ba..0a451401088 100644 --- a/apps/webapp/app/components/dashboard-agent/turn-error.ts +++ b/apps/webapp/app/components/dashboard-agent/turn-error.ts @@ -1,7 +1,8 @@ /** * A failed turn is recorded in the transcript by the agent, under the message id - * `turn-error:{turn}`. The prefix is the transport convention, recognised here so - * the panel can tell a stored failure record apart from an ordinary answer. + * `turn-error:{turn}`. Same arrangement as a wake's `wake:watch:…` id: the prefix + * is the transport convention, recognised here so the panel can tell a stored + * failure record apart from an ordinary answer. * * Live, a failure arrives as the stream's error chunk and `useChat` surfaces it as * the retry callout. The stored record is what a reload reads. Both must never show diff --git a/apps/webapp/app/components/dashboard-agent/unread-counts.test.ts b/apps/webapp/app/components/dashboard-agent/unread-counts.test.ts new file mode 100644 index 00000000000..2405f483b6e --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/unread-counts.test.ts @@ -0,0 +1,76 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { markChatListRead, nextVisibleChat, unreadWorkCount } from "./unread-counts"; + +const list = () => [ + { id: "chat_a", hasUnreadWake: true, hasUnreadWork: true }, + { id: "chat_b", hasUnreadWork: true }, + { id: "chat_c" }, +]; + +/** + * The dot counts chats, not visits. Reading a chat settles it in the list, and the list is + * what the count is derived from — so one visit, or ten, subtracts the same one chat. + */ +describe("the work count is derived from the list", () => { + it("counts every chat still holding unseen work", () => { + expect(unreadWorkCount(list())).toBe(2); + }); + + it("subtracts a read chat once, however many times it is read", () => { + const once = markChatListRead(list(), "chat_a"); + expect(unreadWorkCount(once)).toBe(1); + // The read effect fires on entry and again on cleanup, and again on every revisit. + const again = markChatListRead(markChatListRead(once, "chat_a"), "chat_a"); + expect(unreadWorkCount(again)).toBe(1); + }); + + it("reaches zero only when every chat has been read", () => { + const all = ["chat_a", "chat_b", "chat_c"].reduce(markChatListRead, list()); + expect(unreadWorkCount(all)).toBe(0); + }); + + it("settles the wake alongside the work, so the row stops looking unread", () => { + const read = markChatListRead(list(), "chat_a"); + expect(read[0]).toEqual({ id: "chat_a", hasUnreadWake: false, hasUnreadWork: false }); + // Every other chat is left exactly as it was. + expect(read.slice(1)).toEqual(list().slice(1)); + }); +}); + +/** + * A wake in the chat on screen must not light the dot — but once the panel has let go of that + * chat, its wakes have to reach the dot again. + */ +describe("nextVisibleChat", () => { + it("holds the chat while it is on screen", () => { + expect(nextVisibleChat("chat_a", { leaving: false })).toBe("chat_a"); + }); + + it("lets go on the way out instead of restoring it", () => { + expect(nextVisibleChat("chat_a", { leaving: true })).toBeNull(); + }); +}); + +describe("what the panel and the layout actually do with it", () => { + const panel = readFileSync(new URL("./DashboardAgentPanel.tsx", import.meta.url), "utf8"); + const layout = readFileSync(new URL("./DashboardAgent.tsx", import.meta.url), "utf8"); + + it("reports the count from the list, and only from the list", () => { + expect(panel).toContain("onUnreadWorkChange?.(unreadWorkCount(chats));"); + expect(panel).not.toContain("settled.filter((chat) => chat.hasUnreadWork).length"); + expect(layout).not.toContain("setUnreadWork((count) => Math.max(0, count - 1))"); + }); + + it("tells the read effect's cleanup that it is leaving", () => { + expect(panel).toContain("onChatRead?.(chatId, { leaving: false });"); + expect(panel).toContain("onChatRead?.(chatId, { leaving: true });"); + expect(layout).toContain("visibleChat.current = nextVisibleChat(chatId, options);"); + }); + + it("re-seeds both counts when the environment changes under the layout", () => { + expect(layout).toContain("seededEnvironment.current = environment.id;"); + expect(layout).toContain("setUnreadWakes(initialUnreadWakes);"); + expect(layout).toContain("setUnreadWork(initialUnreadWork);"); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/unread-counts.ts b/apps/webapp/app/components/dashboard-agent/unread-counts.ts new file mode 100644 index 00000000000..08c270af388 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/unread-counts.ts @@ -0,0 +1,30 @@ +/** + * What the launcher's dot counts. + * + * The counts are derived from the chat list rather than nudged up and down as chats are + * opened: a decrement fires once per read effect and once per cleanup, and again on every + * revisit, none of which the server ever hears about. Reading a chat settles it in the list, + * and the list is what the dot counts — so the same chat read twice counts once. + */ + +type UnreadChat = { id: string; hasUnreadWake?: boolean; hasUnreadWork?: boolean }; + +/** + * The chat the panel has on screen. `leaving` is the read effect's cleanup: it runs after the + * panel has already let go, so restoring the id there would keep hiding that chat's wakes. + */ +export function nextVisibleChat(chatId: string, options: { leaving: boolean }): string | null { + return options.leaving ? null : chatId; +} + +/** Opening a chat settles everything unseen in it, not just the wake. */ +export function markChatListRead(chats: T[], chatId: string): T[] { + return chats.map((chat) => + chat.id === chatId ? { ...chat, hasUnreadWake: false, hasUnreadWork: false } : chat + ); +} + +/** How many chats still hold work their owner hasn't seen. */ +export function unreadWorkCount(chats: UnreadChat[]): number { + return chats.filter((chat) => chat.hasUnreadWork).length; +} diff --git a/apps/webapp/app/components/dashboard-agent/unread-work.test.ts b/apps/webapp/app/components/dashboard-agent/unread-work.test.ts new file mode 100644 index 00000000000..9358d5bf335 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/unread-work.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { chatIsUnread } from "./DashboardAgentHistory"; + +/** + * A chat is unread when its transcript moved on after its owner last looked — whether that + * was a watch waking it or an answer that landed while the panel was closed. Both raise the + * dot and the highlight; only a wake also raises a toast. + */ +describe("chatIsUnread", () => { + const chat = (over: Record = {}) => + ({ id: "chat_1", title: "t", lastMessageAt: null, ...over }) as never; + + it("counts work that finished behind a closed panel", () => { + expect(chatIsUnread(chat({ hasUnreadWork: true }))).toBe(true); + }); + + it("still counts a watch wake", () => { + expect(chatIsUnread(chat({ hasUnreadWake: true }))).toBe(true); + }); + + it("leaves a chat its owner has seen", () => { + expect(chatIsUnread(chat())).toBe(false); + expect(chatIsUnread(chat({ hasUnreadWake: false, hasUnreadWork: false }))).toBe(false); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/view-actions.test.ts b/apps/webapp/app/components/dashboard-agent/view-actions.test.ts index a9914da149a..54c2538fc6c 100644 --- a/apps/webapp/app/components/dashboard-agent/view-actions.test.ts +++ b/apps/webapp/app/components/dashboard-agent/view-actions.test.ts @@ -1,7 +1,26 @@ import type { ActionsBlockAction } from "@internal/dashboard-agent-contracts"; import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; -import { answerContinuesAfter, renderableActions } from "./view-actions"; +import { + answerContinuesAfter, + cardAlreadyOffersWatch, + renderableActions, + withoutWatchActions, +} from "./view-actions"; + +const watchAction: ActionsBlockAction = { + label: "Set up a watch", + intent: { + kind: "watch", + spec: { + kind: "error_recurrence", + fingerprint: "a1b2c3", + checkEveryMinutes: 15, + maxHours: 6, + note: "the TypeError in send-order-receipt", + }, + }, +}; const askAction: ActionsBlockAction = { label: "Investigate it", @@ -25,6 +44,10 @@ describe("renderableActions", () => { expect(renderableActions([navigate])).toEqual([navigate]); }); + it("keeps a watch action, spec intact — that spec is what pre-fills the card", () => { + expect(renderableActions([watchAction, askAction])).toEqual([watchAction, askAction]); + }); + it("can filter every action out, leaving nothing to render", () => { expect( renderableActions([{ label: "Nowhere", intent: { kind: "navigate", target: "nope" } }]) @@ -32,6 +55,34 @@ describe("renderableActions", () => { }); }); +describe("one watch button per answer", () => { + const watchAction = { label: "Watch for a repeat", intent: { kind: "watch" as const, spec: {} } }; + const card = (actions: unknown[]) => + ({ type: "investigation", investigation: {}, capabilities: { actions } }) as never; + + it("sees the card's own watch offer", () => { + expect(cardAlreadyOffersWatch([card([watchAction])])).toBe(true); + }); + + it("leaves an answer whose card offers no watch alone", () => { + expect( + cardAlreadyOffersWatch([ + card([{ label: "Keep digging", intent: { kind: "ask", prompt: "" } }]), + ]) + ).toBe(false); + expect(cardAlreadyOffersWatch([])).toBe(false); + }); + + it("drops the model's duplicate offer, keeping everything else", () => { + expect( + withoutWatchActions([ + { label: "Set up a watch", intent: { kind: "watch", spec: {} } }, + { label: "View similar", intent: { kind: "navigate", target: "trigger://x" } }, + ] as never) + ).toEqual([{ label: "View similar", intent: { kind: "navigate", target: "trigger://x" } }]); + }); +}); + describe("keep digging, only while there is digging left", () => { const card = { type: "data-view" }; const text = (t: string) => ({ type: "text", text: t }); @@ -56,7 +107,7 @@ describe("ActionsBlock", () => { }); it("filters through the shared filter rather than rendering every action", () => { - expect(source).toContain("renderableActions(block.actions)"); + expect(source).toContain("renderableActions(actions)"); }); it("is a pure component: no app hooks, no server module, no Remix", () => { diff --git a/apps/webapp/app/components/dashboard-agent/view-actions.ts b/apps/webapp/app/components/dashboard-agent/view-actions.ts index 1d59d20b081..b439d8d96ad 100644 --- a/apps/webapp/app/components/dashboard-agent/view-actions.ts +++ b/apps/webapp/app/components/dashboard-agent/view-actions.ts @@ -4,6 +4,7 @@ import { isTriggerUri, type ActionsBlockAction, type ChartAction, + type ViewBlock, } from "@internal/dashboard-agent-contracts"; type CardAction = ChartAction | ActionsBlockAction; @@ -15,6 +16,23 @@ export function renderableActions(actions: T[]): T[] { }); } +/** + * An investigation card carries its own "watch for a repeat" button, and the model is + * asked to end an unresolved answer with a watch offer — so an answer that does both + * shows the same button twice. The card wins: it is the one with the pre-filled spec. + */ +export function cardAlreadyOffersWatch(blocks: ViewBlock[]): boolean { + return blocks.some( + (block) => + block.type === "investigation" && + (block.capabilities?.actions ?? []).some((action) => action.intent.kind === "watch") + ); +} + +export function withoutWatchActions(actions: T[]): T[] { + return actions.filter((action) => action.intent.kind !== "watch"); +} + /** * "Keep digging" asks the agent to carry on — which is pointless once it already has. * A turn that renders an inconclusive card and then keeps answering leaves the button diff --git a/apps/webapp/app/components/dashboard-agent/view-catalog.tsx b/apps/webapp/app/components/dashboard-agent/view-catalog.tsx index 5d4fb567ed7..78f68fcf3e6 100644 --- a/apps/webapp/app/components/dashboard-agent/view-catalog.tsx +++ b/apps/webapp/app/components/dashboard-agent/view-catalog.tsx @@ -5,6 +5,8 @@ import { InvestigationCard } from "./InvestigationCard"; import { ReportView, type ResolvedUri } from "./ReportView"; import { RunDiagnosisCard } from "./RunDiagnosisCard"; import { blockKey, latestRevisionBlocks } from "./view-blocks"; +import { cardAlreadyOffersWatch } from "./view-actions"; +import { WatchResultBlock } from "./WatchResultBlock"; // Unknown block types are skipped, so an older or newer agent cannot render // arbitrary content. A new block needs a `case` here and a `viewBlockSchema` member. @@ -23,9 +25,11 @@ export function ViewBlocks({ answered?: boolean; }) { if (!Array.isArray(blocks)) return null; + const rendered = latestRevisionBlocks(blocks); + const watchOfferedOnCard = cardAlreadyOffersWatch(rendered); return (
- {latestRevisionBlocks(blocks).map((block) => { + {rendered.map((block) => { // Index into the original array, so collapsing a revision above an // envelope-less block can't shift its key. const key = blockKey(block, blocks.indexOf(block)); @@ -35,7 +39,14 @@ export function ViewBlocks({ case "chart": return ; case "actions": - return ; + return ( + + ); // Revisions share the investigationId, so latest-wins keeps one card. case "investigation": return ( @@ -47,6 +58,9 @@ export function ViewBlocks({ answered={answered} /> ); + // Host-emitted only, so the model cannot fabricate a confirmation. + case "watch_result": + return ; case "report": return ( { + it("still reads the as-built two-value wake id", () => { + expect(wakeRefFromMessageId("wake:watch:watch_1:fired")).toEqual({ + watchId: "watch_1", + outcome: "fired", + }); + expect(wakeRefFromMessageId("wake:watch:watch_1:expired")).toEqual({ + watchId: "watch_1", + outcome: "expired", + }); + expect(wakeRefFromMessageId("msg_1")).toBeNull(); + }); +}); + +describe("wakeResolution", () => { + it("prefers the row's resolution", () => { + expect(wakeResolution("expired", { resolution: "condition_impossible" })).toBe( + "condition_impossible" + ); + }); + + it("reconstructs one for a row written before the resolution column", () => { + expect(wakeResolution("fired", { endedReason: null })).toBe("condition_met"); + expect(wakeResolution("expired", { endedReason: "terminal_unsatisfied" })).toBe( + "condition_impossible" + ); + expect(wakeResolution("expired", { endedReason: "not_met_by_expiry" })).toBe( + "window_completed" + ); + expect(wakeResolution("expired", undefined)).toBe("window_completed"); + }); +}); + +describe("wakePresentation", () => { + it("states the fact, not a generic watch update", () => { + const presented = wakePresentation("fired", { + ...runWatch, + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_SUCCESSFULLY", + durationMs: 4200, + }, + }); + expect(presented.headline).toBe("Run run_abc123 finished"); + expect(presented.label).toBe("Watch update"); + expect(presented.category).toBe("positive"); + }); + + it("shows a failed run as a failure, on the same resolution", () => { + const presented = wakePresentation("fired", { + ...runWatch, + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: null, + }, + }); + expect(presented.headline).toBe("Run run_abc123 failed"); + expect(presented.category).toBe("attention"); + expect(presented.semanticIcon).not.toBe("success"); + }); + + it("names the queue in a drain headline", () => { + expect( + wakePresentation("fired", { + id: "watch_2", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "condition_met", + }).headline + ).toBe("email-sends queue drained"); + }); + + it("reports the threshold watch with its number", () => { + expect( + wakePresentation("fired", { + id: "watch_3", + kind: "queue_depth_above", + identity: "queue_depth_above:email-sends:500", + note: "", + resolution: "condition_met", + observedOutcome: { + kind: "queue_depth_above", + verified: true, + depth: 612, + threshold: 500, + }, + }).headline + ).toBe("email-sends queue is still above 500"); + }); + + it("treats a completed window as an answer, not silence", () => { + const presented = wakePresentation("expired", { + id: "watch_4", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "window_completed", + observedOutcome: { kind: "backlog_drain", verified: true, depth: 42 }, + }); + expect(presented.headline).toBe("email-sends queue is still at 42"); + expect(presented.category).toBe("attention"); + }); + + it("says the condition couldn't be confirmed when the final read failed", () => { + expect( + wakePresentation("expired", { + id: "watch_5", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "window_completed", + observedOutcome: { kind: "backlog_drain", verified: false, depth: null }, + }).headline + ).toBe("The watch ended without a confirmed answer"); + }); + + it("falls back without guessing an outcome when the watch is gone", () => { + const presented = wakePresentation("fired", undefined); + expect(presented.headline).toBe("The watch woke this chat up on its own."); + expect(presented.category).toBe("neutral"); + }); + + it("says an error recurred, and that a quiet window was good news", () => { + const error = { + id: "watch_6", + kind: "error_recurrence", + identity: "error_recurrence:a1b2c3d4e5f6", + note: "", + }; + expect(wakePresentation("fired", { ...error, resolution: "condition_met" })).toMatchObject({ + headline: "Error a1b2c3d4e5f6 happened again", + category: "attention", + }); + expect(wakePresentation("expired", { ...error, resolution: "window_completed" })).toMatchObject( + { headline: "Error a1b2c3d4e5f6 stayed quiet", category: "positive" } + ); + }); + + it("says a queue came back below its threshold, and when it never did", () => { + const below = { + id: "watch_below", + kind: "queue_depth_below", + identity: "queue_depth_below:email-sends:100", + note: "", + }; + expect( + wakePresentation("fired", { + ...below, + resolution: "condition_met", + observedOutcome: { kind: "queue_depth_below", verified: true, depth: 42, threshold: 100 }, + }) + ).toMatchObject({ headline: "email-sends queue is back below 100", category: "positive" }); + + expect( + wakePresentation("expired", { + ...below, + resolution: "window_completed", + observedOutcome: { kind: "queue_depth_below", verified: true, depth: 780, threshold: 100 }, + }) + ).toMatchObject({ headline: "email-sends queue is still above 100", category: "attention" }); + }); + + it("says a queue is stuck at the depth it stalled on, and that it kept moving", () => { + const stalled = { + id: "watch_stalled", + kind: "queue_stalled", + identity: "queue_stalled:email-sends", + note: "", + }; + expect( + wakePresentation("fired", { + ...stalled, + resolution: "condition_met", + observedOutcome: { + kind: "queue_stalled", + verified: true, + depth: 42, + notDecreasingStreak: 3, + ticks: 3, + }, + }) + ).toMatchObject({ headline: "email-sends queue is stuck at 42", category: "attention" }); + + expect( + wakePresentation("expired", { + ...stalled, + resolution: "window_completed", + observedOutcome: { + kind: "queue_stalled", + verified: true, + depth: 3, + notDecreasingStreak: 1, + ticks: 3, + }, + }) + ).toMatchObject({ headline: "email-sends queue kept moving", category: "positive" }); + }); + + it("states the wait and the limit it passed, in minutes", () => { + const age = { + id: "watch_age", + kind: "queue_oldest_age", + identity: "queue_oldest_age:email-sends:5", + note: "", + }; + expect( + wakePresentation("fired", { + ...age, + resolution: "condition_met", + observedOutcome: { + kind: "queue_oldest_age", + verified: true, + ageMs: 12 * 60_000, + thresholdMinutes: 5, + }, + }) + ).toMatchObject({ + headline: "runs in email-sends are waiting 12m (over your 5m limit)", + category: "attention", + }); + + expect( + wakePresentation("expired", { + ...age, + resolution: "window_completed", + observedOutcome: { + kind: "queue_oldest_age", + verified: true, + ageMs: 30_000, + thresholdMinutes: 5, + }, + }) + ).toMatchObject({ headline: "email-sends queue stayed under 5m", category: "positive" }); + }); + + it("names the queue, not the threshold, when a queue-pack watch's queue is gone", () => { + for (const [kind, identity] of [ + ["queue_depth_below", "queue_depth_below:email-sends:100"], + ["queue_stalled", "queue_stalled:email-sends"], + ["queue_oldest_age", "queue_oldest_age:email-sends:5"], + ] as const) { + expect( + wakePresentation("expired", { + id: `watch_${kind}`, + kind, + identity, + note: "", + resolution: "condition_impossible", + }) + ).toMatchObject({ headline: "email-sends queue no longer exists", category: "neutral" }); + } + }); + + it("recovers health without naming an identity", () => { + expect( + wakePresentation("fired", { + id: "watch_7", + kind: "health_recovery", + identity: "health_recovery:health", + note: "", + resolution: "condition_met", + }).headline + ).toBe("Health recovered"); + }); +}); + +describe("watchWakeToastTitle", () => { + const wake = { + watchId: "watch_1", + chatId: "chat_1", + note: "tell me when the nightly invoice run finishes", + }; + + it("leads with the fact, not the notification", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "fired", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + resolution: "condition_met", + }) + ).toBe("email-sends queue drained"); + }); + + it("follows the observed outcome, so a failed run is never good news", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "fired", + kind: "run_finished", + identity: "run_finished:run_abc123", + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 1200, + }, + }) + ).toBe("Run run_abc123 failed"); + }); + + it("reconstructs a resolution for a row written before the model existed", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "expired", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + }) + ).toBe("email-sends queue still hasn't drained"); + }); + + it("claims nothing when the wake carries no watch at all", () => { + expect(watchWakeToastTitle({ ...wake, outcome: "fired" })).toBe( + "The watch woke this chat up on its own." + ); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts b/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts new file mode 100644 index 00000000000..5d38eb98425 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts @@ -0,0 +1,100 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { startWakePolling, UNREAD_POLL_INTERVAL_MS } from "./wake-poll"; + +function harness() { + let hidden = false; + const listeners = new Set<() => void>(); + const loads: number[] = []; + + const stop = startWakePolling({ + load: async () => { + loads.push(Date.now()); + }, + isHidden: () => hidden, + onVisibilityChange: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + // No jitter, so every delay is exactly one interval. + random: () => 0, + setTimer: (callback, ms) => setTimeout(callback, ms) as unknown as number, + clearTimer: (handle) => clearTimeout(handle as unknown as NodeJS.Timeout), + }); + + return { + loads, + stop, + setHidden(next: boolean) { + hidden = next; + for (const listener of listeners) listener(); + }, + listenerCount: () => listeners.size, + }; +} + +describe("startWakePolling", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it("polls once immediately and then once per interval", async () => { + const poll = harness(); + + expect(poll.loads).toHaveLength(1); + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 3); + expect(poll.loads).toHaveLength(4); + + poll.stop(); + }); + + it("asks nothing while hidden and catches up once when visible again", async () => { + const poll = harness(); + poll.setHidden(true); + + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 3); + expect(poll.loads).toHaveLength(1); + + poll.setHidden(false); + await vi.advanceTimersByTimeAsync(0); + expect(poll.loads).toHaveLength(2); + + poll.stop(); + }); + + it("keeps exactly one chain across ten rapid hide/show cycles", async () => { + const poll = harness(); + + for (let cycle = 0; cycle < 10; cycle++) { + poll.setHidden(true); + await vi.advanceTimersByTimeAsync(10); + poll.setHidden(false); + await vi.advanceTimersByTimeAsync(10); + } + + const afterCycles = poll.loads.length; + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 10); + + // One poll per interval, not ten: the resumes replaced the chain instead of + // forking it. + expect(poll.loads.length - afterCycles).toBe(10); + + poll.stop(); + }); + + it("stops every timer and listener on unmount", async () => { + const poll = harness(); + poll.setHidden(true); + poll.setHidden(false); + + poll.stop(); + const settled = poll.loads.length; + + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 10); + expect(poll.loads).toHaveLength(settled); + expect(poll.listenerCount()).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/wake-poll.ts b/apps/webapp/app/components/dashboard-agent/wake-poll.ts new file mode 100644 index 00000000000..c44b252f8f4 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/wake-poll.ts @@ -0,0 +1,68 @@ +/** + * The wake feed's poll: one self-scheduling chain per mount. A hidden tab asks nothing, a + * resume catches up once, and neither can fork the chain into a second one. + */ + +export const UNREAD_POLL_INTERVAL_MS = 60_000; + +// Added to each delay so open tabs never settle into polling on the same second. +export const UNREAD_POLL_JITTER_MS = 15_000; + +export type WakePollOptions = { + load: () => Promise; + isHidden: () => boolean; + /** Subscribe to visibility changes; returns its own unsubscribe. */ + onVisibilityChange: (listener: () => void) => () => void; + /** Seams so a test can drive the chain without real timers. */ + random?: () => number; + setTimer?: (callback: () => void, delayMs: number) => number; + clearTimer?: (handle: number) => void; +}; + +/** Start polling. The returned function stops the chain for good. */ +export function startWakePolling(options: WakePollOptions): () => void { + const random = options.random ?? Math.random; + const setTimer = options.setTimer ?? ((callback, ms) => window.setTimeout(callback, ms)); + const clearTimer = options.clearTimer ?? ((handle) => window.clearTimeout(handle)); + + let stopped = false; + let timer: number | undefined; + let loading = false; + // Each tick carries the chain it belongs to, so an orphaned callback returns + // instead of scheduling itself again. + let chain = 0; + + const tick = (generation: number) => { + if (stopped || generation !== chain) return; + + // Scheduled before the load, so a slow response can't stall the chain. + timer = setTimer( + () => tick(generation), + UNREAD_POLL_INTERVAL_MS + random() * UNREAD_POLL_JITTER_MS + ); + + if (loading || options.isHidden()) return; + loading = true; + const done = () => { + loading = false; + }; + options.load().then(done, done); + }; + + const unsubscribe = options.onVisibilityChange(() => { + if (stopped || options.isHidden()) return; + // One catch-up fetch on a new chain, replacing the pending timer rather than + // adding a second chain. + if (timer !== undefined) clearTimer(timer); + chain += 1; + tick(chain); + }); + + tick(chain); + + return () => { + stopped = true; + if (timer !== undefined) clearTimer(timer); + unsubscribe(); + }; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts b/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts new file mode 100644 index 00000000000..8b25448e318 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts @@ -0,0 +1,136 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type StorageListener = (event: { key: string | null }) => void; + +const store = new Map(); +const storageListeners = new Set(); + +// A minimal `window`: these tests run without a DOM. +const windowStub = { + localStorage: { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => void store.set(key, value), + }, + addEventListener: (_type: string, listener: StorageListener) => + void storageListeners.add(listener), + removeEventListener: (_type: string, listener: StorageListener) => + void storageListeners.delete(listener), +}; + +const { + forgetWatchActivity, + hasWatchActivity, + rememberWatchActivity, + shouldPollWakeFeed, + subscribeWatchActivity, +} = await import("./watch-activity"); + +/** What another tab writing the key looks like here. */ +function otherTabWrote(organizationId: string) { + store.set("tdev:dashboard-agent:watching", JSON.stringify([organizationId])); + for (const listener of storageListeners) listener({ key: "tdev:dashboard-agent:watching" }); +} + +describe("watch activity", () => { + beforeEach(() => { + store.clear(); + storageListeners.clear(); + vi.stubGlobal("window", windowStub); + }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("knows nothing until a watch shows up", () => { + expect(hasWatchActivity("org_1")).toBe(false); + + rememberWatchActivity("org_1"); + expect(hasWatchActivity("org_1")).toBe(true); + expect(hasWatchActivity("org_2")).toBe(false); + }); + + it("survives a reload", () => { + rememberWatchActivity("org_1"); + storageListeners.clear(); + + expect(hasWatchActivity("org_1")).toBe(true); + }); + + it("tells a tab that was already open", () => { + const woken: number[] = []; + const unsubscribe = subscribeWatchActivity(() => woken.push(1)); + + rememberWatchActivity("org_1"); + expect(woken).toHaveLength(1); + + unsubscribe(); + rememberWatchActivity("org_2"); + expect(woken).toHaveLength(1); + }); + + it("tells a tab about a watch another tab created", () => { + const woken: number[] = []; + const unsubscribe = subscribeWatchActivity(() => woken.push(1)); + + otherTabWrote("org_1"); + + expect(woken).toHaveLength(1); + expect(hasWatchActivity("org_1")).toBe(true); + unsubscribe(); + }); + + it("forgets one organization without forgetting the others", () => { + rememberWatchActivity("org_1"); + rememberWatchActivity("org_2"); + + forgetWatchActivity("org_1"); + + expect(hasWatchActivity("org_1")).toBe(false); + expect(hasWatchActivity("org_2")).toBe(true); + }); + + it("remembers at most ten organizations", () => { + for (let index = 0; index < 12; index++) rememberWatchActivity(`org_${index}`); + + expect(hasWatchActivity("org_0")).toBe(false); + expect(hasWatchActivity("org_11")).toBe(true); + }); + + describe("shouldPollWakeFeed", () => { + it("polls in a fresh browser the page load says has an unread wake", () => { + expect(shouldPollWakeFeed({ serverUnreadWakes: 1, organizationId: "org_1" })).toBe(true); + }); + + it("stays quiet when neither the page load nor this browser knows of anything", () => { + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_1" })).toBe(false); + }); + + it("polls in a fresh browser whose only signal is an active watch", () => { + // Created on another machine, nothing woken yet, no local marker. + expect( + shouldPollWakeFeed({ + serverUnreadWakes: 0, + serverHasActiveWatches: true, + organizationId: "org_1", + }) + ).toBe(true); + }); + + it("stays quiet in a fresh browser with no wake and no active watch", () => { + expect( + shouldPollWakeFeed({ + serverUnreadWakes: 0, + serverHasActiveWatches: false, + organizationId: "org_1", + }) + ).toBe(false); + }); + + it("polls without a reload once this browser sees a watch", () => { + rememberWatchActivity("org_1"); + + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_1" })).toBe(true); + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_2" })).toBe(false); + }); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-activity.ts b/apps/webapp/app/components/dashboard-agent/watch-activity.ts new file mode 100644 index 00000000000..e45bf5758a7 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-activity.ts @@ -0,0 +1,94 @@ +/** + * Which organizations this browser has seen agent watches in. This is an accelerator, not the + * gate: a watch created in this tab starts the poll without a reload. The ungated signals are the + * unread count and the active-watch flag the page load carries — see {@link shouldPollWakeFeed}. + * Shared through `localStorage`, so a watch created in one tab wakes the others. + */ + +const STORAGE_KEY = "tdev:dashboard-agent:watching"; + +// Newest ids only, so the key can't grow unbounded. +const MAX_REMEMBERED = 10; + +const listeners = new Set<() => void>(); + +function read(): string[] { + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + return raw ? (JSON.parse(raw) as string[]) : []; + } catch { + // Storage unavailable; treated as "nothing known yet". + return []; + } +} + +export function hasWatchActivity(organizationId: string): boolean { + if (typeof window === "undefined") return false; + return read().includes(organizationId); +} + +/** Called whenever a watch shows up for this org: the poll starts from here. */ +export function rememberWatchActivity(organizationId: string): void { + if (typeof window === "undefined" || hasWatchActivity(organizationId)) return; + try { + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify([...read(), organizationId].slice(-MAX_REMEMBERED)) + ); + } catch { + // Same as the read. This tab still starts polling for the rest of the session. + } + for (const listener of listeners) listener(); +} + +/** + * Called when nothing is left to be woken about. The current tab keeps polling for the rest of + * the session; the next reload starts quiet. + */ +export function forgetWatchActivity(organizationId: string): void { + if (typeof window === "undefined" || !hasWatchActivity(organizationId)) return; + try { + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify(read().filter((id) => id !== organizationId)) + ); + } catch { + // Same as the read. + } +} + +/** + * Whether this browser should poll the wake feed. Both server signals come from the page load, + * so a fresh browser polls without ever opening the panel: `serverUnreadWakes` for a wake that + * already landed, `serverHasActiveWatches` for one created elsewhere that hasn't fired yet. + */ +export function shouldPollWakeFeed(params: { + serverUnreadWakes: number; + serverHasActiveWatches?: boolean; + /** Chats holding work their owner hasn't seen, as the page load counted them. */ + serverUnreadWork?: number; + /** This tab sent a turn that may still be running behind a closed panel. */ + turnInFlight?: boolean; + organizationId: string; +}): boolean { + return ( + params.serverUnreadWakes > 0 || + params.serverHasActiveWatches === true || + (params.serverUnreadWork ?? 0) > 0 || + params.turnInFlight === true || + hasWatchActivity(params.organizationId) + ); +} + +/** Fires when this browser learns of a watch, in this tab or — via `storage` — in another one. */ +export function subscribeWatchActivity(listener: () => void): () => void { + listeners.add(listener); + const onStorage = (event: StorageEvent) => { + if (event.key === null || event.key === STORAGE_KEY) listener(); + }; + window.addEventListener("storage", onStorage); + return () => { + listeners.delete(listener); + window.removeEventListener("storage", onStorage); + }; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts b/apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts new file mode 100644 index 00000000000..9dadd3516cc --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts @@ -0,0 +1,118 @@ +import type { WatchDraft } from "@internal/dashboard-agent-contracts"; +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { NO_WATCH_CARD, watchCardReducer, type WatchCardState } from "./watch-card-state"; + +const draftFor = (note: string): WatchDraft => + ({ + spec: { kind: "error_recurrence", fingerprint: note, checkEveryMinutes: 15, maxHours: 6, note }, + followUp: { investigateOnAttention: false, notifyExternally: false }, + }) as WatchDraft; + +const run = (events: Parameters[1][], from = NO_WATCH_CARD) => + events.reduce(watchCardReducer, from); + +const opened = () => run([{ type: "open", draft: draftFor("the TypeError"), requestId: "wreq_1" }]); + +describe("a watch card belongs to the chat it was configured in", () => { + it("abandons a half-configured card when the chat changes", () => { + expect(run([{ type: "chat-changed" }], opened())).toEqual(NO_WATCH_CARD); + }); + + it("lets go of the request id too, so the next card writes its own records", () => { + const afterFailure = run( + [ + { type: "submitting", requestId: "wreq_1" }, + { type: "failed", error: "nope" }, + ], + opened() + ); + expect(afterFailure.requestId).toBe("wreq_1"); + expect(run([{ type: "chat-changed" }], afterFailure).requestId).toBeUndefined(); + }); + + it("abandons a card that was mid-submit when the chat changed", () => { + const submitting = run([{ type: "submitting", requestId: "wreq_1" }], opened()); + expect(submitting.pending).toBe(true); + expect(run([{ type: "chat-changed" }], submitting)).toEqual(NO_WATCH_CARD); + }); + + it("clears the card once it has been submitted", () => { + expect(run([{ type: "submitted" }], opened())).toEqual(NO_WATCH_CARD); + expect(run([{ type: "dismissed" }], opened())).toEqual(NO_WATCH_CARD); + }); +}); + +describe("the request id survives a retry", () => { + it("keeps the id it was opened with across a failed submit", () => { + const retried = run( + [ + { type: "submitting", requestId: "wreq_1" }, + { type: "failed", error: "nope" }, + { type: "submitting", requestId: "wreq_2" }, + ], + opened() + ); + // A resubmit repairs the same server records; a fresh id would write a second pair. + expect(retried.requestId).toBe("wreq_1"); + expect(retried.error).toBeNull(); + expect(retried.pending).toBe(true); + }); + + it("keeps the edited draft, and edits nothing once the card is gone", () => { + const edited = run([{ type: "edit", draft: draftFor("edited") }], opened()); + expect(edited.draft).toEqual(draftFor("edited")); + expect(edited.requestId).toBe("wreq_1"); + expect(run([{ type: "edit", draft: draftFor("edited") }])).toEqual(NO_WATCH_CARD); + }); + + it("opening a second card starts clean", () => { + const reopened = run( + [ + { type: "failed", error: "nope" }, + { type: "open", draft: draftFor("another"), requestId: "wreq_2" }, + ], + opened() + ); + expect(reopened).toEqual({ + draft: draftFor("another"), + requestId: "wreq_2", + pending: false, + error: null, + }); + }); +}); + +/** + * Structural guard, not behavioural proof: the reducer only sees a chat change if every path + * that changes chat routes through `claimChatSlot`, which is also the only place the in-flight + * open sequence is bumped. + */ +describe("every chat change goes through one door", () => { + const panel = readFileSync(new URL("./DashboardAgentPanel.tsx", import.meta.url), "utf8"); + + it("bumps the open sequence in exactly one place, next to the card reset", () => { + const bumps = panel.match(/openChatRequestSeq\.current\s*(\+\+|\+=)|\+\+openChatRequestSeq/g); + expect(bumps).toHaveLength(1); + expect(panel).toContain( + 'dispatchWatchCard({ type: "chat-changed" });\n return ++openChatRequestSeq.current;' + ); + }); + + it("claims a slot before every setActive that lands in a different chat", () => { + for (const caller of ["openChat", "createChat", "newChat", "submitWatch"]) { + expect(panel).toMatch(new RegExp(`const ${caller} = useCallback\\(`)); + } + // The watch's own landing chat: without the claim, an earlier open still matches its seq. + const submit = panel.slice(panel.indexOf("const submitWatch = useCallback(")); + const claim = submit.indexOf("claimChatSlot();"); + const setActive = submit.indexOf("setActive({ chatId: data.chatId"); + expect(claim).toBeGreaterThan(-1); + expect(setActive).toBeGreaterThan(claim); + }); + + it("leaves no separate watch-draft state for a chat change to miss", () => { + expect(panel).not.toContain("setWatchDraft"); + expect(panel).not.toContain("watchRequestId"); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-card-state.ts b/apps/webapp/app/components/dashboard-agent/watch-card-state.ts new file mode 100644 index 00000000000..59e3c12a0b3 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card-state.ts @@ -0,0 +1,55 @@ +/** + * The panel's watch card, as a pure state machine. + * + * A card is configured against the chat that is open at the time and submitted against + * whatever chat is open when `Start watching` is pressed, so it cannot outlive its chat: + * every chat change abandons it, request id and all. The request id is what makes a retry + * repair the same pair of server records instead of writing a second pair, so it is held + * across a failure and dropped with the card. + */ +import type { WatchDraft } from "@internal/dashboard-agent-contracts"; + +export type WatchCardState = { + draft: WatchDraft | null; + requestId: string | undefined; + pending: boolean; + error: string | null; +}; + +export const NO_WATCH_CARD: WatchCardState = { + draft: null, + requestId: undefined, + pending: false, + error: null, +}; + +export type WatchCardEvent = + | { type: "open"; draft: WatchDraft; requestId: string } + | { type: "edit"; draft: WatchDraft } + | { type: "submitting"; requestId: string } + | { type: "failed"; error: string } + | { type: "submitted" } + | { type: "dismissed" } + | { type: "chat-changed" }; + +export function watchCardReducer(state: WatchCardState, event: WatchCardEvent): WatchCardState { + switch (event.type) { + case "open": + return { draft: event.draft, requestId: event.requestId, pending: false, error: null }; + case "edit": + return state.draft ? { ...state, draft: event.draft } : state; + case "submitting": + return { + ...state, + requestId: state.requestId ?? event.requestId, + pending: true, + error: null, + }; + case "failed": + return { ...state, pending: false, error: event.error }; + case "submitted": + case "dismissed": + case "chat-changed": + return NO_WATCH_CARD; + } +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-card.test.ts b/apps/webapp/app/components/dashboard-agent/watch-card.test.ts new file mode 100644 index 00000000000..de012037b66 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card.test.ts @@ -0,0 +1,388 @@ +import { describe, expect, it } from "vitest"; +import { + WATCH_MAX_QUEUE_AGE_MINUTES, + WATCH_MAX_QUEUE_THRESHOLD, + WATCH_STALL_TICKS_DEFAULT, + watchSpecSchema, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; +import { + clampCadence, + variantsOf, + watchDraftError, + watchDraftFor, + withAgeMinutes, + withCadence, + withFollowUp, + withThreshold, + withVariant, + withWindow, +} from "./watch-card"; +import { + watchConditionLabel, + watchConfirmationBlockBody, + watchDurationLabel, + watchOneShotBlockBody, + watchSubjectLabel, +} from "~/presenters/v3/dashboardAgent"; +import { + errorWatchRecommendation, + healthWatchRecommendation, + queueWatchRecommendation, + runWatchRecommendation, +} from "./watch-recommendations"; + +const queueDraft = () => watchDraftFor(queueWatchRecommendation("email-sends")); +const runDraft = () => watchDraftFor(runWatchRecommendation("run_abc123")); + +describe("the recommendations", () => { + it("gives every entry point a spec the schema accepts", () => { + const specs: WatchSpec[] = [ + runWatchRecommendation("run_abc123"), + queueWatchRecommendation("email-sends"), + errorWatchRecommendation("error_a1b2c3d4"), + healthWatchRecommendation("crit"), + ]; + for (const spec of specs) { + expect(watchSpecSchema.safeParse(spec).success).toBe(true); + } + }); + + it("recommends the condition §2.1 assigns to each object", () => { + expect(runWatchRecommendation("run_abc123").kind).toBe("run_finished"); + expect(queueWatchRecommendation("email-sends").kind).toBe("queue_oldest_age"); + expect(errorWatchRecommendation("error_a1b2c3d4").kind).toBe("error_recurrence"); + expect(healthWatchRecommendation("warn").kind).toBe("health_recovery"); + }); + + it("switches the queue recommendation to the drain once runs are already late", () => { + const late = queueWatchRecommendation("email-sends", { + oldestWaitMs: OLDEST_WAIT_WARNING_MS, + }); + expect(late).toMatchObject({ + kind: "backlog_drain", + queue: "email-sends", + }); + expect(watchSpecSchema.safeParse(late).success).toBe(true); + }); + + it("stays on the age SLA when the queue is merely busy, or the signal is missing", () => { + expect( + queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS - 1 }).kind + ).toBe("queue_oldest_age"); + expect(queueWatchRecommendation("email-sends", { oldestWaitMs: null }).kind).toBe( + "queue_oldest_age" + ); + expect(queueWatchRecommendation("email-sends", {}).kind).toBe("queue_oldest_age"); + expect(queueWatchRecommendation("email-sends").kind).toBe("queue_oldest_age"); + }); + + it("starts both follow-ups off — consent is never assumed", () => { + expect(runDraft().followUp).toEqual({ + investigateOnAttention: false, + notifyExternally: false, + }); + }); +}); + +describe("cadence limits", () => { + it("lets a run watch poll every minute", () => { + expect(clampCadence("run_finished", 1)).toBe(1); + }); + + it("floors an aggregate watch at five minutes — never a hot loop", () => { + expect(clampCadence("backlog_drain", 1)).toBe(5); + expect(clampCadence("queue_depth_above", 1)).toBe(5); + expect(clampCadence("health_recovery", 1)).toBe(5); + }); + + it("keeps an offered cadence and rounds an unknown one up", () => { + expect(clampCadence("backlog_drain", 15)).toBe(15); + expect(clampCadence("backlog_drain", 7)).toBe(15); + expect(clampCadence("backlog_drain", 999)).toBe(60); + }); + + it("re-clamps when the kind changes under the user", () => { + const swapped = withVariant(withCadence(runDraft(), 1), "backlog_drain"); + expect(swapped.spec.checkEveryMinutes).toBe(5); + expect(watchSpecSchema.safeParse(swapped.spec).success).toBe(true); + }); +}); + +describe("condition variants (§3)", () => { + it("offers the run pair and the whole queue family", () => { + expect(variantsOf(runDraft())).toEqual(["run_finished", "run_failed"]); + expect(variantsOf(queueDraft())).toEqual([ + "backlog_drain", + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", + ]); + expect(variantsOf(watchDraftFor(errorWatchRecommendation("error_a1")))).toHaveLength(1); + expect(variantsOf(watchDraftFor(healthWatchRecommendation("warn")))).toHaveLength(1); + }); + + it("carries the subject and window across a swap, and restates the note", () => { + const draft = withWindow(runDraft(), 6); + const failed = withVariant(draft, "run_failed"); + expect(failed.spec).toMatchObject({ + kind: "run_failed", + runId: "run_abc123", + maxHours: 6, + note: "tell me if run run_abc123 fails", + }); + }); + + it("restates the note when the threshold number changes", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + // Same verb and same SLA format as the card's condition line: both come from + // the presenter's one wording record. + expect(above.spec.note).toBe("tell me if the email-sends queue goes above 500"); + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 90); + expect(age.spec.note).toBe("tell me if runs in email-sends wait longer than 1h 30m"); + }); + + it("gives the threshold variant a usable default", () => { + const above = withVariant(queueDraft(), "queue_depth_above"); + expect(above.spec).toMatchObject({ kind: "queue_depth_above", queue: "email-sends" }); + expect(watchDraftError(above)).toBeNull(); + }); + + it("swaps back without losing the queue", () => { + const roundTrip = withVariant(withVariant(queueDraft(), "queue_depth_above"), "backlog_drain"); + expect(roundTrip.spec).toMatchObject({ kind: "backlog_drain", queue: "email-sends" }); + }); + + it("gives every queue variant a submittable default and keeps the subject", () => { + for (const kind of [ + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", + ] as const) { + const swapped = withVariant(queueDraft(), kind); + expect(swapped.spec).toMatchObject({ kind, queue: "email-sends" }); + expect(watchDraftError(swapped)).toBeNull(); + expect(watchSpecSchema.safeParse(swapped.spec).success).toBe(true); + } + }); + + it("carries a typed threshold between the two threshold questions", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + const below = withVariant(above, "queue_depth_below"); + expect(below.spec).toMatchObject({ kind: "queue_depth_below", threshold: 500 }); + }); + + it("keeps the stall count internal — the default, never a field", () => { + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(stalled.spec).toMatchObject({ ticks: WATCH_STALL_TICKS_DEFAULT }); + expect(withThreshold(stalled, 5)).toEqual(stalled); + expect(withAgeMinutes(stalled, 5)).toEqual(stalled); + }); +}); + +describe("the window", () => { + it("never leaves the 24-hour ceiling", () => { + expect(withWindow(runDraft(), 999).spec.maxHours).toBe(24); + }); + + it("never goes below the shortest offered window", () => { + expect(withWindow(runDraft(), 0).spec.maxHours).toBe(0.5); + }); +}); + +describe("the follow-up opt-ins (§2.2, binding)", () => { + it("sets them INDEPENDENTLY — never as a radio group", () => { + const both = withFollowUp(withFollowUp(runDraft(), { notifyExternally: true }), { + investigateOnAttention: true, + }); + expect(both.followUp).toEqual({ investigateOnAttention: true, notifyExternally: true }); + }); + + it("turning one off leaves the other alone", () => { + const draft = withFollowUp(runDraft(), { + investigateOnAttention: true, + notifyExternally: true, + }); + expect(withFollowUp(draft, { notifyExternally: false }).followUp).toEqual({ + investigateOnAttention: true, + notifyExternally: false, + }); + }); + + it("has no way to express in-chat delivery at all — it is not a choice", () => { + expect(Object.keys(runDraft().followUp).sort()).toEqual([ + "investigateOnAttention", + "notifyExternally", + ]); + }); +}); + +describe("validation stays inside the card", () => { + it("accepts every recommendation as it opens", () => { + expect(watchDraftError(runDraft())).toBeNull(); + expect(watchDraftError(queueDraft())).toBeNull(); + }); + + it("refuses a half-typed threshold", () => { + const draft = withThreshold(withVariant(queueDraft(), "queue_depth_above"), Number.NaN); + expect(watchDraftError(draft)).toMatch(/whole number/i); + }); + + it("refuses a threshold above the queue-watch ceiling", () => { + const draft = withThreshold( + withVariant(queueDraft(), "queue_depth_above"), + WATCH_MAX_QUEUE_THRESHOLD + 1 + ); + expect(watchDraftError(draft)).toMatch(/too high/i); + }); + + it("ignores a threshold set on a kind that has none", () => { + expect(withThreshold(runDraft(), 5)).toEqual(runDraft()); + }); + + it("refuses a half-typed threshold on the `below` variant too", () => { + const draft = withThreshold(withVariant(queueDraft(), "queue_depth_below"), Number.NaN); + expect(watchDraftError(draft)).toMatch(/whole number/i); + }); + + it("refuses an SLA that is empty, zero, or longer than a watch can run", () => { + const age = withVariant(queueDraft(), "queue_oldest_age"); + expect(watchDraftError(withAgeMinutes(age, Number.NaN))).toMatch(/whole number of minutes/i); + expect(watchDraftError(withAgeMinutes(age, 0))).toMatch(/whole number of minutes/i); + expect(watchDraftError(withAgeMinutes(age, WATCH_MAX_QUEUE_AGE_MINUTES + 1))).toMatch( + /longer than a watch can run/i + ); + expect(watchDraftError(withAgeMinutes(age, 30))).toBeNull(); + }); + + it("ignores an SLA set on a kind that has none", () => { + expect(withAgeMinutes(runDraft(), 5)).toEqual(runDraft()); + }); +}); + +describe("the card's copy", () => { + it("names the subject the way the object does", () => { + expect(watchSubjectLabel(queueWatchRecommendation("email-sends"))).toBe("email-sends"); + expect(watchSubjectLabel(runWatchRecommendation("run_abc123"))).toBe("run run_abc123"); + expect(watchSubjectLabel(healthWatchRecommendation("warn"))).toBe("health"); + }); + + it("says the kind once, and names the error in full", () => { + // Fingerprints are stored prefixed (`error_c4b4a797397a9c43`), so the raw value + // would read "error error_c4b4a797397a9c43". + expect( + watchSubjectLabel({ + kind: "error_recurrence", + fingerprint: "error_c4b4a797397a9c43", + checkEveryMinutes: 5, + maxHours: 0.5, + }) + ).toBe("error c4b4a797397a9c43"); + }); + + it("states the condition and the duration as §2.2 writes them", () => { + const spec = queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS }); + expect(watchConditionLabel(spec)).toBe("Until the queue drains"); + expect(watchDurationLabel(spec)).toBe("For 1 hour · checking every 5 min"); + }); + + it("carries the threshold into the condition line", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + expect(watchConditionLabel(above.spec)).toBe("If the queue goes above 500"); + }); + + it("states each new queue condition the way the user reads it", () => { + const below = withThreshold(withVariant(queueDraft(), "queue_depth_below"), 100); + expect(watchConditionLabel(below.spec)).toBe("Until the queue is back below 100"); + + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(watchConditionLabel(stalled.spec)).toBe("If the queue stops moving"); + + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 90); + expect(watchConditionLabel(age.spec)).toBe("If runs wait longer than 1h 30m"); + expect(watchSubjectLabel(age.spec)).toBe("email-sends"); + }); + + it("writes the confirmation as one sentence for every queue condition", () => { + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(watchConfirmationBlockBody({ spec: stalled.spec, watchId: "w" }).headline).toBe( + "Watching email-sends in case it stops moving." + ); + + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 5); + expect(watchConfirmationBlockBody({ spec: age.spec, watchId: "w" }).headline).toBe( + "Watching email-sends in case runs wait longer than 5m." + ); + + const below = withThreshold(withVariant(queueDraft(), "queue_depth_below"), 100); + expect(watchConfirmationBlockBody({ spec: below.spec, watchId: "w" }).headline).toBe( + "Watching email-sends until it is back below 100." + ); + }); +}); + +describe("the persisted blocks (§2.2)", () => { + it("states all four lifetime facts on a confirmation", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS }), + watchId: "watch_1", + }); + expect(body.outcome).toBe("watching"); + expect(body.headline).toBe("Watching email-sends until the queue drains."); + expect(body.lifetime).toBe( + "Checking every 5 min for up to 1 hour. It reports once, then stops." + ); + expect(body.watchId).toBe("watch_1"); + expect(body.detail).toBeNull(); + }); + + it("says plainly when the creation-time check couldn't run", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + unavailable: true, + }); + expect(body.detail).toBe("We couldn't check that just now. Watching anyway."); + }); + + it("only claims a follow-up that actually took effect", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + followUp: { investigateOnAttention: true, external: { status: "not_requested" } }, + }); + expect(body.followUp).toEqual(["If it turns out badly, I'll investigate straight away."]); + }); + + it("says out loud when the email the user asked for couldn't be added", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + followUp: { external: { status: "unavailable", reason: "email_alerts_not_configured" } }, + }); + expect(body.followUp).toEqual([ + "I couldn't add email notifications, so updates will appear in the dashboard only.", + ]); + }); + + it("makes a one-shot result carry no lifetime and no watch", () => { + const satisfied = watchOneShotBlockBody({ + spec: queueWatchRecommendation("email-sends"), + result: "satisfied", + }); + expect(satisfied.outcome).toBe("already_true"); + expect(satisfied.headline).toBe("That already happened, so there's nothing left to watch."); + expect(satisfied.lifetime).toBeNull(); + expect(satisfied.watchId).toBeNull(); + + const impossible = watchOneShotBlockBody({ + spec: runWatchRecommendation("run_abc123"), + result: "terminal_unsatisfied", + }); + expect(impossible.outcome).toBe("impossible"); + expect(impossible.headline).toBe("That can't happen any more, so there's nothing to watch."); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-card.ts b/apps/webapp/app/components/dashboard-agent/watch-card.ts new file mode 100644 index 00000000000..9d2a9d54425 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card.ts @@ -0,0 +1,181 @@ +/** + * The watch card's state machine, kept pure so its rules are testable without a DOM. + * + * The card never invents a value the schema would reject: switching condition + * variant re-clamps the cadence, and the window is always one of the offered + * options. The option lists are read from contracts (`watchCadenceOptions`, + * `WATCH_WINDOW_HOURS_OPTIONS`) rather than re-typed, so a picker cannot offer + * something validation would refuse. Nothing here persists: a draft is client-side + * until `Start watching` submits it. + */ +import { + WATCH_DEFAULT_QUEUE_AGE_MINUTES, + WATCH_DEFAULT_QUEUE_THRESHOLD, + WATCH_MAX_HOURS, + WATCH_MAX_QUEUE_AGE_MINUTES, + WATCH_MAX_QUEUE_THRESHOLD, + WATCH_STALL_TICKS_DEFAULT, + WATCH_WINDOW_HOURS_OPTIONS, + watchCadenceOptions, + watchConditionVariants, + watchSpecSchema, + type WatchDraft, + type WatchFollowUp, + type WatchKind, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { noteFor } from "~/presenters/v3/dashboardAgent"; + +/** A brand-new draft: the recommendation, with both opt-ins off. */ +export function watchDraftFor(spec: WatchSpec): WatchDraft { + return { spec, followUp: { investigateOnAttention: false, notifyExternally: false } }; +} + +/** + * The nearest cadence this kind is allowed to poll at. A 1-minute run watch + * switched to a queue variant must land on 5, not fail validation on submit. + */ +export function clampCadence(kind: WatchKind, minutes: number): number { + const options = watchCadenceOptions(kind); + if (options.includes(minutes)) return minutes; + return options.find((option) => option >= minutes) ?? options[options.length - 1]!; +} + +/** + * Swap the condition for its sibling variant, carrying everything else except the + * note, which is restated to describe the new condition. + */ +export function withVariant(draft: WatchDraft, kind: WatchKind): WatchDraft { + const next = variantSpec(draft, kind); + if (next === draft.spec) return draft; + return { ...draft, spec: { ...next, note: noteFor(next) } as WatchSpec }; +} + +function variantSpec(draft: WatchDraft, kind: WatchKind): WatchSpec { + const { spec } = draft; + const common = { + note: spec.note, + maxHours: spec.maxHours, + checkEveryMinutes: clampCadence(kind, spec.checkEveryMinutes), + } as const; + + switch (kind) { + case "run_finished": + case "run_failed": + case "run_start": { + const runId = "runId" in spec ? spec.runId : ""; + return { ...common, kind, runId } as WatchSpec; + } + case "backlog_drain": { + const queue = "queue" in spec ? spec.queue : ""; + return { ...common, kind, queue } as WatchSpec; + } + case "queue_depth_above": + case "queue_depth_below": { + const queue = "queue" in spec ? spec.queue : ""; + // The number carries across the two threshold questions: someone who typed + // 500 for "above" means the same 500 when they flip to "back below". + const threshold = "threshold" in spec ? spec.threshold : WATCH_DEFAULT_QUEUE_THRESHOLD; + return { ...common, kind, queue, threshold } as WatchSpec; + } + case "queue_stalled": { + const queue = "queue" in spec ? spec.queue : ""; + // Ticks are not user-facing: the card never shows a field for them. + const ticks = "ticks" in spec ? spec.ticks : WATCH_STALL_TICKS_DEFAULT; + return { ...common, kind, queue, ticks } as WatchSpec; + } + case "queue_oldest_age": { + const queue = "queue" in spec ? spec.queue : ""; + const thresholdMinutes = + "thresholdMinutes" in spec ? spec.thresholdMinutes : WATCH_DEFAULT_QUEUE_AGE_MINUTES; + return { ...common, kind, queue, thresholdMinutes } as WatchSpec; + } + // The kinds with no second question keep the draft untouched. + default: + return draft.spec; + } +} + +/** + * The conditions this draft's picker offers, in order, including the current one. + * A single-entry list means the kind has no second question and the card states + * the condition as a fact instead of a choice. + */ +export function variantsOf(draft: WatchDraft): readonly WatchKind[] { + return watchConditionVariants(draft.spec.kind); +} + +export function withCadence(draft: WatchDraft, minutes: number): WatchDraft { + return { + ...draft, + spec: { + ...draft.spec, + checkEveryMinutes: clampCadence(draft.spec.kind, minutes), + } as WatchSpec, + }; +} + +export function withWindow(draft: WatchDraft, maxHours: number): WatchDraft { + const clamped = Math.min(Math.max(maxHours, WATCH_WINDOW_HOURS_OPTIONS[0]), WATCH_MAX_HOURS); + return { ...draft, spec: { ...draft.spec, maxHours: clamped } as WatchSpec }; +} + +/** + * The threshold, as the user is typing it. No range checks here: a half-typed + * field is a draft, and `watchDraftError` is what refuses to submit it. + */ +export function withThreshold(draft: WatchDraft, threshold: number): WatchDraft { + if (draft.spec.kind !== "queue_depth_above" && draft.spec.kind !== "queue_depth_below") { + return draft; + } + // The note quotes the number, so a new number restates the note. + const spec = { ...draft.spec, threshold }; + return { ...draft, spec: { ...spec, note: noteFor(spec) } }; +} + +/** The age SLA in minutes, as the user is typing it. Same rule as the threshold. */ +export function withAgeMinutes(draft: WatchDraft, thresholdMinutes: number): WatchDraft { + if (draft.spec.kind !== "queue_oldest_age") return draft; + // The note quotes the number, so a new number restates the note. + const spec = { ...draft.spec, thresholdMinutes }; + return { ...draft, spec: { ...spec, note: noteFor(spec) } }; +} + +/** + * The two follow-up opt-ins, set independently. There is no way to express + * "external instead of chat": in-chat delivery is not a choice, so it is not here. + */ +export function withFollowUp(draft: WatchDraft, patch: Partial): WatchDraft { + return { ...draft, followUp: { ...draft.followUp, ...patch } }; +} + +/** + * Why this draft can't be submitted, in the user's words, or null when it can. + * The schema is the authority, so the card and the server agree by construction; + * this only translates its refusal into the sentence the card shows inline. + */ +export function watchDraftError(draft: WatchDraft): string | null { + if (draft.spec.kind === "queue_depth_above" || draft.spec.kind === "queue_depth_below") { + const { threshold } = draft.spec; + if (!Number.isInteger(threshold) || threshold < 0) { + return "Enter a whole number to watch for."; + } + if (threshold > WATCH_MAX_QUEUE_THRESHOLD) { + return `That threshold is too high — ${WATCH_MAX_QUEUE_THRESHOLD.toLocaleString()} is the most a queue watch takes.`; + } + } + + if (draft.spec.kind === "queue_oldest_age") { + const { thresholdMinutes } = draft.spec; + if (!Number.isInteger(thresholdMinutes) || thresholdMinutes < 1) { + return "Enter a whole number of minutes to watch for."; + } + if (thresholdMinutes > WATCH_MAX_QUEUE_AGE_MINUTES) { + return `That's longer than a watch can run — ${WATCH_MAX_QUEUE_AGE_MINUTES} minutes is the most.`; + } + } + + return watchSpecSchema.safeParse(draft.spec).success + ? null + : "Something in this watch isn't valid. Check the duration and the condition."; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts b/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts new file mode 100644 index 00000000000..687bc12cda4 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts @@ -0,0 +1,107 @@ +import { watchIdentity, type WatchSpec } from "@internal/dashboard-agent-contracts"; +import { describe, expect, it } from "vitest"; +import { immediateWatchMessage, watchChipLabel, watchChipTooltip } from "./watch-chips"; + +const chip = (spec: WatchSpec) => ({ + kind: spec.kind, + identity: watchIdentity(spec), + note: spec.note, +}); + +describe("watchChipLabel", () => { + it("labels a run watch with its run id", () => { + expect( + watchChipLabel( + chip({ + kind: "run_finished", + runId: "run_abc123", + note: "Tell me when the retry finishes.", + maxHours: 2, + checkEveryMinutes: 1, + }) + ) + ).toBe("run_abc123"); + }); + + it("labels a backlog watch with the queue name", () => { + expect( + watchChipLabel( + chip({ + kind: "backlog_drain", + queue: "task/send-email", + note: "Tell me when the backlog clears.", + maxHours: 6, + checkEveryMinutes: 5, + }) + ) + ).toBe("task/send-email"); + }); + + it("labels an error watch by its fingerprint, in full", () => { + expect( + watchChipLabel( + chip({ + kind: "error_recurrence", + fingerprint: "0123456789abcdef0123456789abcdef", + note: "Tell me if the rate-limit error comes back.", + maxHours: 12, + checkEveryMinutes: 15, + }) + ) + ).toBe("0123456789abcdef0123456789abcdef"); + }); + + it("labels a health watch by its kind, not its report", () => { + expect( + watchChipLabel( + chip({ + kind: "health_recovery", + report: "health", + fromSeverity: "crit", + note: "prod health back to normal", + maxHours: 4, + checkEveryMinutes: 15, + }) + ) + ).toBe("health"); + }); + + it("falls back to the first words of the note when the identity is unreadable", () => { + expect( + watchChipLabel({ kind: "run_start", identity: "nonsense", note: "Tell me when it starts" }) + ).toBe("Tell me when"); + }); + + it("falls back to the kind when there is no note either", () => { + expect(watchChipLabel({ kind: "run_start", identity: "", note: " " })).toBe("run_start"); + }); +}); + +describe("watchChipTooltip", () => { + it("carries the note, the cadence and the state", () => { + expect( + watchChipTooltip({ + note: "Tell me when prod recovers.", + checkEveryMinutes: 15, + status: "active", + }) + ).toBe("Tell me when prod recovers. · every 15 min · watching"); + }); + + it("drops an empty note rather than leaving a dangling separator", () => { + expect(watchChipTooltip({ note: "", checkEveryMinutes: 5, status: "fired" })).toBe( + "every 5 min · fired" + ); + }); +}); + +describe("immediateWatchMessage", () => { + it("says the condition already resolved", () => { + expect(immediateWatchMessage("satisfied")).toMatch(/already happened/); + expect(immediateWatchMessage("terminal_unsatisfied")).toMatch(/can't happen any more/); + }); + + it("never falls through to nothing", () => { + expect(immediateWatchMessage("something-new")).toBeTruthy(); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-chips.ts b/apps/webapp/app/components/dashboard-agent/watch-chips.ts new file mode 100644 index 00000000000..c8de7a3eed9 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-chips.ts @@ -0,0 +1,76 @@ +/** + * The pure text of the watch UI: a chip's label and its tooltip. + * + * A chip has one line of room in a 380px panel, so the label names the thing being + * watched and the icon carries the state. The label comes from the watch `identity`, + * the same dedup key the store uses, so a chip cannot disagree with the store about + * what it watches. + */ +import type { WatchStatus } from "@internal/dashboard-agent-contracts"; + +// The immediate-check wording lives in the presenter with the rest of the +// user-facing copy. Re-exported here for chip callers. +export { immediateWatchMessage } from "~/presenters/v3/dashboardAgent"; + +import { + formatWatchCadence, + shortFingerprint, + watchIdentityValue, +} from "~/presenters/v3/dashboardAgent"; + +export const WATCH_STATUS_LABEL: Record = { + active: "watching", + fired: "fired", + expired: "expired", + cancelled: "cancelled", +}; + +/** Fingerprints are hashes — a chip shows just enough of one to tell them apart. */ +/** + * The chip label for a watch. `identity` is `{kind}:{value}`, so the value is the + * thing being watched; a health watch has no per-instance value, so its kind is + * the label. Falls back to the note (then the kind) if the identity is unreadable. + */ +export function watchChipLabel(watch: { kind: string; identity: string; note: string }): string { + const value = watch.identity.startsWith(`${watch.kind}:`) + ? watch.identity.slice(watch.kind.length + 1) + : ""; + + switch (watch.kind) { + case "run_start": + case "run_finished": + case "run_failed": + case "backlog_drain": + case "queue_stalled": + return value || fallbackLabel(watch); + // Identity is `{kind}:{queue}:{number}` here. The chip names the queue; the + // number goes in the tooltip's note, where there is room for it. + case "queue_depth_above": + case "queue_depth_below": + case "queue_oldest_age": + return watchIdentityValue(watch.kind, watch.identity) || fallbackLabel(watch); + case "error_recurrence": + return value ? shortFingerprint(value) : fallbackLabel(watch); + case "health_recovery": + return "health"; + default: + return value || fallbackLabel(watch); + } +} + +/** Last resort: the first few words of the note, else the kind as written. */ +function fallbackLabel(watch: { kind: string; note: string }): string { + const words = watch.note.trim().split(/\s+/).filter(Boolean).slice(0, 3).join(" "); + return words || watch.kind; +} + +/** Everything that didn't fit on the chip: why it exists, and its cadence. */ +export function watchChipTooltip(watch: { + note: string; + checkEveryMinutes: number; + status: WatchStatus; +}): string { + const note = watch.note.trim(); + const cadence = formatWatchCadence(watch.checkEveryMinutes); + return [note, cadence, WATCH_STATUS_LABEL[watch.status]].filter(Boolean).join(" · "); +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts b/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts new file mode 100644 index 00000000000..4ab53c2dc90 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts @@ -0,0 +1,78 @@ +import { + WATCH_DEFAULT_QUEUE_AGE_MINUTES, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; +import { noteFor } from "~/presenters/v3/dashboardAgent"; + +/** Distributes over the spec union, so the kind stays discriminated. */ +type WithoutNote = T extends unknown ? Omit : never; + +/** The note comes from the presenter, so a recommendation reads like an edited one. */ +function withNote(spec: WithoutNote): WatchSpec { + const draft = { ...spec, note: "" } as WatchSpec; + return { ...draft, note: noteFor(draft) }; +} + +export function runWatchRecommendation(runFriendlyId: string): WatchSpec { + return withNote({ + kind: "run_finished", + runId: runFriendlyId, + checkEveryMinutes: 1, + maxHours: 1, + }); +} + +/** + * The recommendation must be a condition that isn't true yet: an already-true watch + * one-shots instead of watching. Past the wait threshold that means the drain, not the SLA. + */ +export function queueWatchRecommendation( + queueName: string, + context?: { oldestWaitMs?: number | null } +): WatchSpec { + const oldestWaitMs = context?.oldestWaitMs ?? null; + if (oldestWaitMs !== null && oldestWaitMs >= OLDEST_WAIT_WARNING_MS) { + return withNote({ + kind: "backlog_drain", + queue: queueName, + checkEveryMinutes: 5, + maxHours: 1, + }); + } + + return queueAgeWatchRecommendation(queueName); +} + +export function queueAgeWatchRecommendation( + queueName: string, + thresholdMinutes: number = WATCH_DEFAULT_QUEUE_AGE_MINUTES +): WatchSpec { + return withNote({ + kind: "queue_oldest_age", + queue: queueName, + thresholdMinutes, + checkEveryMinutes: 5, + maxHours: 1, + }); +} + +export function errorWatchRecommendation(errorFriendlyId: string): WatchSpec { + return withNote({ + kind: "error_recurrence", + fingerprint: errorFriendlyId, + checkEveryMinutes: 5, + maxHours: 6, + }); +} + +/** Only offered on a degraded report. `fromSeverity` is what the recovery is measured from. */ +export function healthWatchRecommendation(fromSeverity: "warn" | "crit"): WatchSpec { + return withNote({ + kind: "health_recovery", + report: "health", + fromSeverity, + checkEveryMinutes: 5, + maxHours: 2, + }); +} diff --git a/apps/webapp/app/components/queues/queue-name.ts b/apps/webapp/app/components/queues/queue-name.ts new file mode 100644 index 00000000000..baf9c60a3d6 --- /dev/null +++ b/apps/webapp/app/components/queues/queue-name.ts @@ -0,0 +1,7 @@ +/** + * `TaskQueue.name` as the engine and the watch checks store it. A task queue keeps its + * `task/` prefix there, and the presenters strip it for display only. + */ +export function storedQueueName(queue: { type: string; name: string }): string { + return queue.type === "task" ? `task/${queue.name.replace(/^task\//, "")}` : queue.name; +} diff --git a/apps/webapp/app/components/queues/queue-thresholds.ts b/apps/webapp/app/components/queues/queue-thresholds.ts index 43cf2be32e7..1c714bc4035 100644 --- a/apps/webapp/app/components/queues/queue-thresholds.ts +++ b/apps/webapp/app/components/queues/queue-thresholds.ts @@ -1,2 +1,2 @@ -/** Head-of-line wait at which a queue reads as stuck. */ +/** Head-of-line wait at which a queue reads as stuck. Shared by the queue page and the watch card. */ export const OLDEST_WAIT_WARNING_MS = 5 * 60_000; diff --git a/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts index 83ab09c177c..dff916a6aa1 100644 --- a/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts @@ -18,6 +18,7 @@ export const ApiAlertType = z.enum([ "deployment_failure", "deployment_success", "error_group", + "dashboard_agent_watch", ]); export type ApiAlertType = z.infer; @@ -88,6 +89,8 @@ export class ApiAlertChannelPresenter { return "deployment_success"; case "ERROR_GROUP": return "error_group"; + case "DASHBOARD_AGENT_WATCH": + return "dashboard_agent_watch"; default: assertNever(alertType); } @@ -105,6 +108,8 @@ export class ApiAlertChannelPresenter { return "DEPLOYMENT_SUCCESS"; case "error_group": return "ERROR_GROUP"; + case "dashboard_agent_watch": + return "DASHBOARD_AGENT_WATCH"; default: assertNever(alertType); } diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts b/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts new file mode 100644 index 00000000000..df3c791a82e --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts @@ -0,0 +1,98 @@ +/** + * A view block or a resolved watch as plain text. + * + * The panel renders blocks as React; an email, a Slack message, a webhook body or + * a log line cannot. Rather than each of those re-saying the block's contents in + * its own words, they render it here. Pure, no React, no request context. + */ +import type { ViewBlock } from "@internal/dashboard-agent-contracts"; +import { presentResolvedWatch, watchNoteLine, type WatchResolvedInput } from "./watch-wording"; + +/** A labelled scalar the check observed. */ +export type TextFact = { label: string; value: string }; + +/** Facts as one `Label: value` line each. */ +export function renderFactLines(facts: readonly TextFact[]): string[] { + return facts.map((fact) => `${fact.label}: ${fact.value}`); +} + +function lines(...parts: Array): string { + return parts + .filter((part): part is string => typeof part === "string" && part.length > 0) + .join("\n"); +} + +/** + * One view block as plain text. Says what the card says and nothing more — no + * surface may add a sentence of its own on top. + */ +export function renderBlockAsText(block: ViewBlock): string { + switch (block.type) { + case "watch_result": + return lines(block.headline, block.lifetime, block.detail, ...block.followUp); + + case "diagnosis": + return lines( + block.summary, + `Likely cause: ${block.likelyCause}`, + `Confidence: ${block.confidence}`, + block.impact ? `Impact: ${block.impact}` : null, + ...block.evidence.map( + (item) => + `Evidence (${item.type}): ${item.detail}${item.reference ? ` — ${item.reference}` : ""}` + ), + ...block.nextSteps.map((step, index) => `${index + 1}. ${step}`) + ); + + case "investigation": { + const state = block.investigation; + return lines( + state.title, + state.headline, + `Outcome: ${state.outcome} · severity ${state.severity} · confidence ${state.confidence}`, + ...state.hypotheses.map( + (hypothesis) => + `${hypothesis.statement} — ${hypothesis.verdict}${ + hypothesis.finding ? `: ${hypothesis.finding}` : "" + }` + ), + state.remediation ? `Fix: ${state.remediation}` : null, + ...(state.checkNext ?? []).map((step) => `Check next: ${step}`), + state.caveat ? `Caveat: ${state.caveat.message}` : null + ); + } + + case "report": { + const { vm } = block; + return lines( + `${vm.title} report for ${vm.scope} (${vm.period}): ${vm.summary.severity}`, + ...vm.findings.map((finding) => `${finding.type} — ${finding.severity}: ${finding.reason}`) + ); + } + + // A chart is its shape, not its rows: the rows come from running the query. + case "chart": + return lines(`Chart: ${block.title ?? "untitled"} (${block.chartType})`, block.query); + + case "actions": + return lines(...block.actions.map((action) => `- ${action.label}`)); + + default: { + const unreachable: never = block; + throw new Error(`Unhandled view block: ${JSON.stringify(unreachable)}`); + } + } +} + +/** + * A resolved watch as plain text: the fact, why it was being watched, then what the + * resolving check saw. What the email body and the Slack message both say. + */ +export function renderResolvedWatchAsText(args: { + resolved: WatchResolvedInput; + note: string; + facts: readonly TextFact[]; +}): string { + const { headline } = presentResolvedWatch(args.resolved); + return lines(headline, watchNoteLine(args.note), ...renderFactLines(args.facts)); +} diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/index.ts b/apps/webapp/app/presenters/v3/dashboardAgent/index.ts new file mode 100644 index 00000000000..07d781dc6db --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/index.ts @@ -0,0 +1,5 @@ +// The dashboard agent's presenter: the one place a watch, a view block or a watch +// result becomes English. Every surface (card, banner, toast, email, Slack, +// webhook) imports from here. +export * from "./block-text"; +export * from "./watch-wording"; diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts b/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts new file mode 100644 index 00000000000..a0da0585206 --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts @@ -0,0 +1,39 @@ +/** + * The watch vocabulary moved into the contracts package so the agent's own + * deterministic narration says the same sentences the dashboard does — the agent + * cannot import the webapp, and a second vocabulary would drift within a release. + * + * Re-exported here because every webapp surface imports the presenter, not contracts. + */ +export { + formatWatchCadence, + formatWatchDuration, + formatWatchSla, + formatWatchWait, + formatWatchWindow, + immediateWatchMessage, + noteFor, + presentResolvedWatch, + WATCH_IN_CHAT_DELIVERY_LINE, + WATCH_PRESENTATION_FALLBACK, + WATCH_UPDATE_LABEL, + shortFingerprint, + watchConditionLabel, + watchConditionWording, + watchConfirmationBlockBody, + watchDurationLabel, + watchExternalNotificationLine, + watchFollowUpLines, + watchIdentityValue, + watchLifetimeSentence, + watchNoteLine, + watchOneShotBlockBody, + watchRequestSentence, + watchSubjectLabel, + watchSubline, + watchTooltipLabel, + type WatchConditionWording, + type WatchPresentation, + type WatchResolvedInput, + type WatchSemanticIcon, +} from "@internal/dashboard-agent-contracts"; diff --git a/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts b/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts index d6a8f2ebd9c..fbe416db338 100644 --- a/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts @@ -11,7 +11,7 @@ import { type ReportViewModel } from "./report-view-model"; const DEFAULT_PERIOD = "1h"; -/** How long a finished report stays reusable. */ +/** How long a finished report stays reusable. Must stay under the watch tick cadence. */ export const REPORT_CACHE_TTL_MS = 90_000; /** How many report, environment and period triples one instance keeps. */ diff --git a/apps/webapp/app/presenters/v3/reports/report-layout.ts b/apps/webapp/app/presenters/v3/reports/report-layout.ts index c9605bc96b1..3597c8b5829 100644 --- a/apps/webapp/app/presenters/v3/reports/report-layout.ts +++ b/apps/webapp/app/presenters/v3/reports/report-layout.ts @@ -50,12 +50,35 @@ export const REPORT_LABELS = { read: "read:", /** The footer heading. */ nextSteps: "Next steps", - /** Shown beside the report's name when the data can't be trusted. */ - staleBadge: "stale data", - staleNote: - "The telemetry behind this report is stale, so the numbers below are informational only.", } as const; +/** The flag beside the report's name, and the caveat under its headline. */ +export type LayoutTrust = { badge: string; note: string }; + +/** + * Why a report's numbers can't be trusted, in its own words. Stale, absent and unmeasured are three + * different states, and a snapshot with no telemetry feed must not be called stale. + */ +const TRUST_CAVEATS: Record = { + telemetry_stale: { + badge: "stale data", + note: "The telemetry behind this report is stale, so the numbers below are informational only.", + }, + telemetry_absent: { + badge: "no telemetry", + note: "No telemetry reached this report, so the numbers below are a point-in-time snapshot rather than a measured window.", + }, + flow_unmeasured: { + badge: "unmeasured", + note: "Throughput could not be measured over this window, so the numbers below are informational only.", + }, +}; + +const TRUST_CAVEAT_FALLBACK: LayoutTrust = { + badge: "unverified data", + note: "The data behind this report could not be verified, so the numbers below are informational only.", +}; + /** * The report's sections, top to bottom. A renderer walks this order; a new section has to be added * here first, which is what keeps the surfaces aligned. `trust` spans two places: a flag beside the @@ -87,13 +110,20 @@ const UNASSESSABLE_REASONS = new Set(["unknown", "flow_unmeasured"]); const NEUTRAL_REASONS = new Set(["freshness_unknown", "flow_unmeasured"]); /** - * `facts.trustworthy === false` means the telemetry behind the verdict is stale, so the numbers are - * informational only. Absent = trustworthy (the common case, and what pre-`facts` snapshots imply). + * `facts.trustworthy === false` means the numbers behind the verdict are informational only. Absent + * = trustworthy (the common case, and what pre-`facts` snapshots imply). */ export function reportIsTrustworthy(vm: { facts?: Record }): boolean { return vm.facts?.trustworthy !== false; } +/** The caveat for an untrustworthy report, chosen by `facts.untrustworthyReason`. */ +export function reportTrust(vm: { facts?: Record }): LayoutTrust | undefined { + if (reportIsTrustworthy(vm)) return undefined; + const reason = vm.facts?.untrustworthyReason; + return (typeof reason === "string" ? TRUST_CAVEATS[reason] : undefined) ?? TRUST_CAVEAT_FALLBACK; +} + export function reportTone(severity: Severity, reason?: string): ReportTone { return reason !== undefined && NEUTRAL_REASONS.has(reason) ? "neutral" : severity; } @@ -283,7 +313,7 @@ export type LayoutFooterEntry = { export type ReportLayout = { header: { name: string; meta: string }; /** Present only when the data can't be trusted. */ - trust?: { badge: string; note: string }; + trust?: LayoutTrust; headline: { tone: ReportTone; glyph: string; severity: Severity; phrase: string; text?: string }; /** The finding the headline speaks for, always expanded. */ hero?: LayoutFinding; @@ -343,14 +373,14 @@ export function buildReportLayout(vm: LayoutViewModel, messages: ReportMessages) .filter((finding) => finding.read !== undefined && !UNASSESSABLE_REASONS.has(finding.reason)) .map((finding) => fillTokens(messages.readMessage(finding.read!), tokens)); + const trust = reportTrust(vm); + return { header: { name: vm.title, meta: [vm.scope, vm.period, vm.baselineLabel].filter(Boolean).join(" · "), }, - ...(reportIsTrustworthy(vm) - ? {} - : { trust: { badge: REPORT_LABELS.staleBadge, note: REPORT_LABELS.staleNote } }), + ...(trust === undefined ? {} : { trust }), headline: { severity: vm.summary.severity, tone: reportTone(vm.summary.severity, heroStatement?.reason), @@ -518,12 +548,29 @@ function metricValue(metric: LayoutMetricInput, messages: ReportMessages): strin : fmtValue(metric.value, metric.unit); } +/** + * How far a metric fell below its baseline: `undefined` when the fall doesn't round past 1×, and + * `null` when it collapsed to nothing and no multiplier can say it. + */ +function fallMultiplier(metric: LayoutMetricInput): number | null | undefined { + if (metric.normal === undefined || metric.normal <= 0) return undefined; + if (metric.value <= 0) return null; + const fall = Math.round(metric.normal / metric.value); + return fall > 1 ? fall : undefined; +} + /** * A metric's movement against its baseline. A multiplier only reads as movement once it rounds past * 1×; below that a metric with a baseline is flat, and one without has nothing to compare against. */ function metricDelta(metric: LayoutMetricInput): LayoutDelta | undefined { const delta = metric.delta; + // A fall's own multiplier rounds to 0 or 1, so measure how far it fell instead. + if (delta?.dir === "down") { + const fall = fallMultiplier(metric); + if (fall === null) return { text: REPORT_GLYPH.down, dir: "down" }; + if (fall !== undefined) return { text: `${REPORT_GLYPH.down} ${fall}×`, dir: "down" }; + } if (delta && delta.mult !== undefined && delta.mult > 1 && delta.dir !== "flat") { return { text: `${delta.dir === "up" ? REPORT_GLYPH.up : REPORT_GLYPH.down} ${delta.mult}×`, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx index e0ee26255b7..41dd31c5d17 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx @@ -53,9 +53,13 @@ export const meta = pageMeta("New alert"); const FormSchema = z .object({ alertTypes: z - .array(z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"])) + .array( + z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS", "DASHBOARD_AGENT_WATCH"]) + ) .min(1) - .or(z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"])), + .or( + z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS", "DASHBOARD_AGENT_WATCH"]) + ), environmentTypes: z .array(z.enum(["STAGING", "PRODUCTION", "PREVIEW"])) .min(1) @@ -456,6 +460,18 @@ export default function Page() { defaultChecked /> +
+ + +
+ {alertTypes.errors} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx index a2d77e99edc..7ce5500a875 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx @@ -570,6 +570,8 @@ export function alertTypeTitle(alertType: ProjectAlertType): string { return "Deployment success"; case "ERROR_GROUP": return "Error group"; + case "DASHBOARD_AGENT_WATCH": + return "Dashboard agent watches"; default: { throw new Error(`Unknown alertType: ${alertType}`); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index ff69c89c52b..1946312aac3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -24,6 +24,8 @@ import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; import { RunsIcon } from "~/assets/icons/RunsIcon"; import { CodeBlock } from "~/components/code/CodeBlock"; import { InvestigateButton } from "~/components/dashboard-agent/InvestigateButton"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { errorWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; import { errorGroupPrompt } from "~/components/dashboard-agent/investigate-prompts"; import { ErrorStatusBadge } from "~/components/errors/ErrorStatusBadge"; import { @@ -586,7 +588,7 @@ function ErrorDetailSidebar({
Details - {/* Self-hides when the agent isn't available. */} + {/* Both buttons self-hide when the agent isn't available. */}
+
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 38434f1cdf6..8eb34adc51f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -8,6 +8,9 @@ import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { AnimatedOrgBannerBar } from "~/components/billing/AnimatedOrgBannerBar"; import { BigNumber } from "~/components/metrics/BigNumber"; import { Header3 } from "~/components/primitives/Headers"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { queueWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; +import { storedQueueName } from "~/components/queues/queue-name"; import { OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { Spinner } from "~/components/primitives/Spinner"; @@ -121,7 +124,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } const queue = retrieve.queue; - const fullName = queue.type === "task" ? `task/${queue.name}` : queue.name; + const fullName = storedQueueName(queue); const maxPeriodDays = await queueMetricsMaxPeriodDays(environment.organizationId); @@ -331,14 +334,20 @@ export default function Page() { maxPeriodDays={maxPeriodDays} shortcut={{ key: "d" }} /> - {/* Self-hides when the agent isn't available. */} + {/* Both buttons self-hide when the agent isn't available. Watch is + pre-filled with this queue's recommendation. */} {degraded ? ( ) : null} + {/* A paused queue can't drain or grow, so every watch it could offer is a + promise nothing will keep until someone resumes it. */} + {queue.paused ? null : ( + + )} { }) : null; + // One narrow read per page load, so the wake signal reaches a browser that has never opened + // the panel — including one whose watch hasn't fired yet. The poll never asks for this. + let dashboardAgentActivity: DashboardAgentWakeActivity = { + unreadWakes: 0, + hasActiveWatches: false, + }; + let dashboardAgentUnreadWork = 0; + if (hasDashboardAgentAccess) { + try { + [dashboardAgentActivity, dashboardAgentUnreadWork] = await Promise.all([ + readDashboardAgentWakeActivity(dashboardAgentDb, { + organizationId: project.organization.id, + userId: user.id, + }), + countChatsWithUnreadWork(dashboardAgentDb, { + organizationId: project.organization.id, + userId: user.id, + }), + ]); + } catch (error) { + // The dashboard must load even when the agent's store doesn't answer. + logger.error("Failed to read dashboard agent wake activity", { error }); + } + } + return { ...project, hasDashboardAgentAccess, promotedDashboardAgentPrompt, + dashboardAgentActivity, + dashboardAgentUnreadWork, }; }; export default function Page() { - const { hasDashboardAgentAccess, promotedDashboardAgentPrompt } = useLoaderData(); + const { + hasDashboardAgentAccess, + promotedDashboardAgentPrompt, + dashboardAgentActivity, + dashboardAgentUnreadWork, + } = useLoaderData(); return ( diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts new file mode 100644 index 00000000000..10cf14d6abb --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts @@ -0,0 +1,100 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { resolveAgentAlertContext } from "~/services/dashboardAgentAlertContext.server"; +import { unsubscribeChannelFromWatchAlerts } from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; + +/** + * `DELETE /api/v1/dashboard-agent/alerts/:channelId` — stop alerting this channel + * when a watch fires. The channel is looked up scoped to the chat's project. + */ + +const ParamsSchema = z.object({ channelId: z.string().min(1) }); + +const BodySchema = z.object({ + chatId: z.string().min(1), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "DELETE") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const authentication = await authenticateUatOrApiRequest(request); + if (!authentication?.userActor) { + return json({ error: "Invalid or missing access token" }, { status: 401 }); + } + if (authentication.userActor.client !== "dashboard-agent") { + return json({ error: "Not allowed", code: "forbidden_client" }, { status: 403 }); + } + const userId = authentication.userActor.userId; + // The turn's environment scope is the authority for the chat's project below. + const environmentId = authentication.userActor.environmentId; + if (!environmentId) { + return json( + { error: "This chat has no environment context.", code: "invalid_target" }, + { status: 400 } + ); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + const body = parsedBody.data; + + try { + const context = await resolveAgentAlertContext({ + userId, + environmentId, + chatId: body.chatId, + claimedEnvironmentId: body.environmentId, + claimedProjectRef: body.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: context.code === "environment_mismatch" ? 400 : 404 } + ); + } + + const result = await unsubscribeChannelFromWatchAlerts(parsedParams.data.channelId, { + projectId: context.environment.project.id, + // A project is shared by every member, so the caller's own address is part of the scope. + organizationId: context.environment.organizationId, + ownerUserId: userId, + }); + if (!result.ok) { + if (result.reason === "conflict") { + return json( + { error: "That alert was being changed elsewhere. Try again.", code: "conflict" }, + { status: 409 } + ); + } + return json({ error: "Alert not found", code: "not_found" }, { status: 404 }); + } + + return json({ ok: true, disabledChannel: result.disabledChannel }); + } catch (error) { + logger.error("Failed to unsubscribe a channel from dashboard agent watch alerts", { + error, + userId, + environmentId, + channelId: parsedParams.data.channelId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts new file mode 100644 index 00000000000..cde67806e67 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts @@ -0,0 +1,220 @@ +import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { $replica, prisma } from "~/db.server"; +import { + ProjectAlertEmailProperties, + ProjectAlertSlackProperties, +} from "~/models/projectAlert.server"; +import { + resolveAgentAlertContext, + type AgentAlertContextError, +} from "~/services/dashboardAgentAlertContext.server"; +import { + canUseDashboardAgentEmailAlerts, + DASHBOARD_AGENT_WATCH_ALERT_TYPE, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; +import { CreateAlertChannelService } from "~/v3/services/alerts/createAlertChannel.server"; + +/** + * `GET` lists this chat's project's watch alerts; `POST` subscribes the user's email. Only + * the agent's delegated user-actor token is accepted, and the environment comes from it. + */ + +const ListQuerySchema = z.object({ + chatId: z.string().min(1), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +const CreateBodySchema = z.object({ + chatId: z.string().min(1), + channel: z.literal("email"), + /** May only be the authenticated user's own account email. */ + email: z.string().email().optional(), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +/** A token without an environment scope is unusable here. */ +async function authenticate( + request: Request +): Promise<{ userId: string; environmentId: string } | { error: Response }> { + const authentication = await authenticateUatOrApiRequest(request); + const actor = authentication?.userActor; + if (!actor || actor.client !== "dashboard-agent") { + return { error: json({ error: "Invalid or missing access token" }, { status: 401 }) }; + } + if (!actor.environmentId) { + return { + error: json( + { error: "This chat has no environment context.", code: "invalid_target" }, + { status: 400 } + ), + }; + } + return { userId: actor.userId, environmentId: actor.environmentId }; +} + +/** A mismatched claim is the caller's error, the rest are 404s. */ +function contextStatus(code: AgentAlertContextError) { + return code === "environment_mismatch" ? 400 : 404; +} + +export async function loader({ request }: LoaderFunctionArgs) { + const auth = await authenticate(request); + if ("error" in auth) return auth.error; + + const query = ListQuerySchema.safeParse( + Object.fromEntries(new URL(request.url).searchParams.entries()) + ); + if (!query.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const context = await resolveAgentAlertContext({ + userId: auth.userId, + environmentId: auth.environmentId, + chatId: query.data.chatId, + claimedEnvironmentId: query.data.environmentId, + claimedProjectRef: query.data.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: contextStatus(context.code) } + ); + } + + const channels = await $replica.projectAlertChannel.findMany({ + where: { + projectId: context.environment.project.id, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + }, + select: { id: true, type: true, enabled: true, properties: true, environmentTypes: true }, + orderBy: { createdAt: "asc" }, + }); + + return json({ + alerts: channels.map((channel) => ({ + id: channel.id, + type: channel.type, + enabled: channel.enabled, + environmentTypes: channel.environmentTypes, + target: describeTarget(channel.type, channel.properties), + })), + }); +} + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const auth = await authenticate(request); + if ("error" in auth) return auth.error; + const { userId } = auth; + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const parsed = CreateBodySchema.safeParse(rawBody); + if (!parsed.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + const body = parsed.data; + + const context = await resolveAgentAlertContext({ + userId, + environmentId: auth.environmentId, + chatId: body.chatId, + claimedEnvironmentId: body.environmentId, + claimedProjectRef: body.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: contextStatus(context.code) } + ); + } + const { environment } = context; + + const gate = await canUseDashboardAgentEmailAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + projectId: environment.project.id, + }); + if (!gate.allowed) { + return json({ error: "Alerts are not available here", code: gate.reason }, { status: 403 }); + } + + // Only the signed-in user's own account email may be subscribed. Read off the + // primary: this is the identity the subscription is pinned to. + const user = await prisma.user.findFirst({ where: { id: userId }, select: { email: true } }); + if (!user) { + return json({ error: "User not found", code: "invalid_request" }, { status: 404 }); + } + const email = user.email; + if (body.email && body.email.trim().toLowerCase() !== email.toLowerCase()) { + return json( + { + error: + "Watch alerts can only go to your own account email. Ask the user to add another address on the Alerts page.", + code: "email_not_allowed", + }, + { status: 400 } + ); + } + + try { + const service = new CreateAlertChannelService(); + const channel = await service.call(environment.project.externalRef, userId, { + name: `Watch alerts for ${email}`, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE], + environmentTypes: [environment.type], + // Stable per (email, project), so asking twice re-enables one channel. + deduplicationKey: `dashboard-agent-watch:${email}`, + channel: { type: "EMAIL", email }, + }); + + return json({ id: channel.id, type: channel.type, target: email, enabled: channel.enabled }); + } catch (error) { + // A thrown Response is Remix control flow, not a failure to report. + if (error instanceof Response) throw error; + logger.error("Failed to create a dashboard agent watch alert channel", { + error, + userId, + organizationId: environment.organizationId, + projectId: environment.project.id, + environmentId: environment.id, + }); + return json({ error: "Internal Server Error", code: "internal" }, { status: 500 }); + } +} + +/** A short, non-secret description of where a channel delivers. */ +function describeTarget(type: string, properties: unknown): string | undefined { + if (type === "EMAIL") { + const parsed = ProjectAlertEmailProperties.safeParse(properties); + return parsed.success ? maskEmail(parsed.data.email) : undefined; + } + if (type === "SLACK") { + const parsed = ProjectAlertSlackProperties.safeParse(properties); + return parsed.success ? `#${parsed.data.channelName}` : undefined; + } + // Webhook URLs stay out of the agent's context entirely. + return undefined; +} + +function maskEmail(email: string): string { + const [local, domain] = email.split("@"); + if (!domain || !local) return "an email address"; + const head = local.slice(0, 2); + return `${head}${local.length > 2 ? "…" : ""}@${domain}`; +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts new file mode 100644 index 00000000000..406f0b5b2e5 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts @@ -0,0 +1,191 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { cancelWatch, getWatch, recordWatchCheck } from "@internal/dashboard-agent-db"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { checkWatch, previousCheckFacts } from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { + armDashboardAgentWatchBatch, + authorizeWatchEnvironment, +} from "~/services/dashboardAgentWatches.server"; +import { + WATCH_TOKEN_GRACE_MS, + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * Private per-watch check. The token only names a watch; the row is the authority on + * lifecycle and its snapshot, and this route transitions nothing and advances no tick. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +/** Best-effort: a chain that couldn't be armed returns `false` and is retried next check. */ +async function ensureBatchChain(watch: { + id: string; + environmentId: string; + spec: { checkEveryMinutes: number }; +}): Promise { + try { + const { running } = await armDashboardAgentWatchBatch({ + environmentId: watch.environmentId, + cadenceMinutes: watch.spec.checkEveryMinutes, + }); + return running; + } catch (error) { + logger.error("Dashboard agent watch check: couldn't arm the batch chain", { + watchId: watch.id, + environmentId: watch.environmentId, + error, + }); + return false; + } +} + +const BodySchema = z.object({ + /** The expiry evaluation: allowed after `expiresAt`, within the token's grace. */ + final: z.boolean().optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + // A valid token for a different watch is 403, not 401. + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + let rawBody: unknown; + try { + const raw = await request.text(); + rawBody = raw.length > 0 ? JSON.parse(raw) : {}; + } catch { + return json({ error: "Invalid request body" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) return json({ error: "Invalid request body" }, { status: 400 }); + const body = parsedBody.data; + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + // Terminal watches are never checked again, whatever the token says. + if (watch.status !== "active") { + return json( + { + error: `This watch is ${watch.status}`, + code: watch.status === "cancelled" ? "cancelled" : "not_active", + status: watch.status, + }, + { status: 403 } + ); + } + + const now = new Date(); + const expired = watch.expiresAt.getTime() <= now.getTime(); + if (expired) { + // Past the deadline only the final evaluation is allowed, inside the token's grace. + const graceEnds = watch.expiresAt.getTime() + WATCH_TOKEN_GRACE_MS; + if (body.final !== true || now.getTime() > graceEnds) { + return json( + { error: "This watch has expired", code: "expired", expiresAt: watch.expiresAt }, + { status: 403 } + ); + } + } + + try { + // Re-authorize the initiating user before any environment data is read. + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + // A watch must not outlive the access it was created with. Never notified. + await cancelWatch(dashboardAgentDb, { id: watchId, reason: "access_revoked" }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { status: 403 } + ); + } + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const outcome = await checkWatch( + watch.spec, + watchCheckDeps(authorization.environment, now), + // A tick that couldn't read anything freezes a streak instead of resetting it. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch check failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }) + ); + + // Recorded even on the final evaluation. Guarded on `active`, so a concurrent + // fire/expire wins and this no-ops. + await recordWatchCheck(dashboardAgentDb, { + id: watchId, + lastResult: { + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + final: body.final === true, + }, + }); + + const batched = await ensureBatchChain(watch); + + // `observed` travels with the verdict so no delivery surface re-reads the source. + return json({ + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + batched, + }); + } catch (error) { + logger.error("Dashboard agent watch check tick failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts new file mode 100644 index 00000000000..427435835c1 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts @@ -0,0 +1,113 @@ +import { + claimWatchAlertDispatch, + getWatch, + releaseWatchAlertDispatch, +} from "@internal/dashboard-agent-db"; +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { enqueueWatchFiredAlert } from "~/services/dashboardAgentWatchAlerts.server"; +import { authorizeWatchEnvironment } from "~/services/dashboardAgentWatches.server"; +import { + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; +import { logger } from "~/services/logger.server"; + +/** + * The watcher task reports a fired watch. The row is the authority on whether it fired, and + * the initiating user is re-authorized against its snapshot before any alert is sent. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + // Anything that isn't a fired watch gets no alert, whatever the caller claims. + if (watch.status !== "fired" || !watch.firedAt) { + return json( + { error: `This watch is ${watch.status}`, code: "not_fired", status: watch.status }, + { status: 409 } + ); + } + + try { + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + // Not cancelled here: the watch is already terminal. + logger.info("Dashboard agent watch fired, but access was revoked; no alert", { watchId }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { + status: 403, + } + ); + } + + const claimed = await claimWatchAlertDispatch(dashboardAgentDb, { + id: watch.id, + terminalStatus: "fired", + }); + if (!claimed) { + logger.info("Dashboard agent watch fired callback repeated; no second alert", { watchId }); + return json({ ok: true, alerted: false }); + } + + try { + await enqueueWatchFiredAlert(watch, "fired"); + } catch (error) { + await releaseWatchAlertDispatch(dashboardAgentDb, { id: watch.id, terminalStatus: "fired" }); + throw error; + } + + return json({ ok: true, alerted: true }); + } catch (error) { + logger.error("Dashboard agent watch fire callback failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts new file mode 100644 index 00000000000..83af5eba382 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts @@ -0,0 +1,105 @@ +import { getWatch, isTerminalWatchStatus } from "@internal/dashboard-agent-db"; +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { + kickWatchInvestigation, + watchWantsInvestigation, +} from "~/services/dashboardAgentWatchInvestigate.server"; +import { authorizeWatchEnvironment } from "~/services/dashboardAgentWatches.server"; +import { + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; +import { logger } from "~/services/logger.server"; + +/** + * The watcher task reports a delivered wake for a pre-approved investigation. The caller's + * body is ignored: consent, outcome, user and environment all come off the row. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + if (!isTerminalWatchStatus(watch.status)) { + return json( + { error: `This watch is ${watch.status}`, code: "not_resolved", status: watch.status }, + { status: 409 } + ); + } + + if (!watchWantsInvestigation(watch)) { + // No consent, or an outcome consent doesn't cover: the wake was the whole delivery. + return json({ ok: true, investigating: false }); + } + + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + logger.info("Dashboard agent watch resolved, but access was revoked; no investigation", { + watchId, + }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { status: 403 } + ); + } + + // Never an error to the caller: the wake is already delivered and marked, so a failed + // kick must not make the watcher retry. The stale-investigation sweep settles it. + try { + await kickWatchInvestigation({ watch, environment: authorization.environment }); + } catch (error) { + // A thrown Response is Remix control flow, not a failed kick. + if (error instanceof Response) throw error; + logger.error("Dashboard agent watch investigation could not be started", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + return json({ ok: true, investigating: false, code: "kick_failed" }); + } + + return json({ ok: true, investigating: true }); +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts new file mode 100644 index 00000000000..ac869d68ea5 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts @@ -0,0 +1,75 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { runWatchBatchCheck } from "~/services/dashboardAgentWatchBatch.server"; +import { logger } from "~/services/logger.server"; +import { + bearerToken, + verifyWatchBatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * Private batch check: one call per (environment, cadence) group per cadence. The token + * names the group, the body names the tick. `runWatchBatchCheck` documents the rest. + */ + +const BodySchema = z.object({ + environmentId: z.string().min(1), + cadenceMinutes: z.number().int().positive(), + epoch: z.number().int().nonnegative(), + tick: z.number().int().positive(), +}); + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchBatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + let rawBody: unknown; + try { + const raw = await request.text(); + rawBody = raw.length > 0 ? JSON.parse(raw) : {}; + } catch { + return json({ error: "Invalid request body" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) return json({ error: "Invalid request body" }, { status: 400 }); + const body = parsedBody.data; + + // A valid token for a different group is 403, not 401. + if ( + claims.environmentId !== body.environmentId || + claims.cadenceMinutes !== body.cadenceMinutes + ) { + return json({ error: "Not allowed for this group", code: "group_mismatch" }, { status: 403 }); + } + + try { + return json(await runWatchBatchCheck(body)); + } catch (error) { + logger.error("Dashboard agent watch batch check failed", { + error, + environmentId: body.environmentId, + cadenceMinutes: body.cadenceMinutes, + epoch: body.epoch, + tick: body.tick, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts new file mode 100644 index 00000000000..0fbe83e3463 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts @@ -0,0 +1,163 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { watchSpecSchema } from "@internal/dashboard-agent-contracts"; +import { z } from "zod"; +import { logger } from "~/services/logger.server"; +import { resolveWatchEmailAlertsState } from "~/services/dashboardAgentWatchAlerts.server"; +import { + authorizeWatchEnvironmentById, + createDashboardAgentWatch, + resolveChatWatchContext, +} from "~/services/dashboardAgentWatches.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; + +/** + * Programmatic watch creation (MCP). Only the agent's delegated user-actor token is + * accepted, and the environment comes from it, never the body or the chat's stored context. + */ + +const BodySchema = z.object({ + spec: watchSpecSchema, + chatId: z.string().min(1), + /** Consent for the wake turn to open an investigation. Off unless explicitly sent. */ + investigateOnAttention: z.boolean().optional(), + /** + * Only checked against the token's environment scope, never used in its place. + * `environmentId` is the canonical `RuntimeEnvironment.id`, not a slug. + */ + projectRef: z.string().min(1).optional(), + environmentId: z.string().min(1).optional(), +}); + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const authentication = await authenticateUatOrApiRequest(request); + if (!authentication?.userActor) { + return json({ error: "Invalid or missing access token" }, { status: 401 }); + } + if (authentication.userActor.client !== "dashboard-agent") { + return json({ error: "Not allowed", code: "forbidden_client" }, { status: 403 }); + } + const userId = authentication.userActor.userId; + // The environment this turn is scoped to. There is no trusted fallback. + const environmentId = authentication.userActor.environmentId; + if (!environmentId) { + return json( + { error: "This chat has no environment context to watch in.", code: "invalid_target" }, + { status: 400 } + ); + } + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid watch request", code: "invalid_request" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) { + return json({ error: "Invalid watch request", code: "invalid_request" }, { status: 400 }); + } + const parsed = parsedBody.data; + + // Refuse a body naming a different environment rather than silently picking one. + if (parsed.environmentId && parsed.environmentId !== environmentId) { + return json( + { + error: "That environment isn't the one this chat is open in.", + code: "environment_mismatch", + }, + { status: 400 } + ); + } + + try { + // A chat this user doesn't own does not exist here. + const chat = await resolveChatWatchContext({ chatId: parsed.chatId, userId }); + if (!chat) { + return json({ error: "Chat not found", code: "chat_not_found" }, { status: 404 }); + } + + // The same authorization a background check applies. + const environment = await authorizeWatchEnvironmentById({ userId, environmentId }); + if (!environment) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + // A chat belongs to one org; its watches can't point at another org's env. + if (environment.organizationId !== chat.organizationId) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + // Same check as `environmentId`, for callers that send the project instead. + if (parsed.projectRef && environment.project.externalRef !== parsed.projectRef) { + return json( + { + error: "That project isn't the one this chat is open in.", + code: "environment_mismatch", + }, + { status: 400 } + ); + } + + const result = await createDashboardAgentWatch({ + environment, + userId, + chatId: parsed.chatId, + spec: parsed.spec, + investigateOnAttention: parsed.investigateOnAttention, + }); + + if (!result.ok) { + const status = + result.code === "limit_reached" || result.code === "duplicate" + ? 409 + : result.code === "invalid_target" + ? 404 + : // The chat was deleted while the create was in flight. + result.code === "chat_not_found" + ? 404 + : result.code === "not_configured" + ? 501 + : 500; + return json( + { + error: result.error, + code: result.code, + ...(result.existingId ? { existingId: result.existingId } : {}), + }, + { status } + ); + } + + // One-shot: the immediate check answered, so there is no watch row and no id. + if (!result.watching) { + return json({ + watching: false, + identity: result.identity, + immediate: { result: result.immediate.result, facts: result.immediate.facts }, + }); + } + + return json({ + watching: true, + watchId: result.watchId, + identity: result.identity, + status: result.status, + expiresAt: result.expiresAt.toISOString(), + emailAlerts: await resolveWatchEmailAlertsState({ userId, environment }), + ...(result.unavailable ? { unavailable: true } : {}), + }); + } catch (error) { + // A thrown Response is Remix control flow, not a failure to report. + if (error instanceof Response) throw error; + logger.error("Failed to create a dashboard agent watch", { + error, + userId, + environmentId, + chatId: parsed.chatId, + }); + return json({ error: "Internal Server Error", code: "internal" }, { status: 500 }); + } +} diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.workers.$tagName.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.workers.$tagName.ts index 31b0f486d31..33cb38a1045 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.workers.$tagName.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.workers.$tagName.ts @@ -70,6 +70,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { triggerSource: true, createdAt: true, payloadSchema: true, + queueConfig: true, }, orderBy: { slug: "asc", @@ -100,6 +101,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { triggerSource: task.triggerSource, createdAt: task.createdAt, payloadSchema: task.payloadSchema, + queueConfig: task.queueConfig, })), }, urls, diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.metrics.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.metrics.ts index f63977d416a..da3ad91744a 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.metrics.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.metrics.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { logger } from "~/services/logger.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { queueDepthSeries } from "~/v3/queueDepthSeries"; /** * Per-queue metrics over a window. `queueParam` is the queue name; `?type=task` (the default) @@ -62,8 +63,12 @@ export const loader = createLoaderApiRoute( const windowMinutes = windowMs / 60_000; const bucketSeconds = Math.max(60, Math.round(windowMs / 1000 / TREND_POINTS)); // Snap both bounds to the bucket grid so repeated calls share ClickHouse cache entries. - const endMs = Math.ceil(Date.now() / (bucketSeconds * 1000)) * bucketSeconds * 1000; + const bucketIntervalMs = bucketSeconds * 1000; + const endMs = Math.ceil(Date.now() / bucketIntervalMs) * bucketIntervalMs; const startMs = endMs - windowMs; + // The trend grid covers whole buckets, so a period that isn't a bucket multiple still lines up. + const gridStartMs = Math.floor(startMs / bucketIntervalMs) * bucketIntervalMs; + const numBuckets = Math.round((endMs - gridStartMs) / bucketIntervalMs); try { const clickhouse = await clickhouseFactory.getClickhouseForOrganization( @@ -115,12 +120,13 @@ export const loader = createLoaderApiRoute( startedCount, startedPerMin: Number((startedCount / windowMinutes).toFixed(2)), throttledCount: summary?.throttled_count ?? 0, - bucketIntervalMs: bucketSeconds * 1000, - // Oldest first; buckets with no sample are omitted, so gaps carry the previous depth. - depthTrend: (trendRows ?? []) - .slice() - .sort((a, b) => a.bucket.localeCompare(b.bucket)) - .map((row) => row.depth), + bucketIntervalMs, + // Oldest first, one point per bucket: a bucket with no sample carries the previous depth. + depthTrend: queueDepthSeries(trendRows ?? [], { + startMs: gridStartMs, + bucketIntervalMs, + numBuckets, + }).depth, }); } catch (error) { // Rethrow Responses: swallowing one would turn it into a 500. diff --git a/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx b/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx new file mode 100644 index 00000000000..771d326e556 --- /dev/null +++ b/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx @@ -0,0 +1,164 @@ +import { EnvelopeIcon } from "@heroicons/react/24/solid"; +import { Form, useNavigation } from "@remix-run/react"; +import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { typedjson, useTypedActionData, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { AppContainer, MainCenteredContainer } from "~/components/layout/AppLayout"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { FormTitle } from "~/components/primitives/FormTitle"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { prisma } from "~/db.server"; +import { verifyUnsubscribeToken } from "~/services/dashboardAgentAlertUnsubscribeToken.server"; +import { + DASHBOARD_AGENT_WATCH_ALERT_TYPE, + unsubscribeChannelFromWatchAlerts, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { rootPath } from "~/utils/pathBuilder"; + +/** + * The unsubscribe link in a watch alert email. The signed token is the whole authorization + * and names one channel; GET confirms and POST acts, so a link preview can't unsubscribe. + */ + +const ParamsSchema = z.object({ channelId: z.string().min(1) }); + +async function authorize(request: Request, params: Record) { + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return undefined; + + const token = new URL(request.url).searchParams.get("token"); + if (!token) return undefined; + + const claims = await verifyUnsubscribeToken(token); + if (!claims) return undefined; + if (claims.channelId !== parsedParams.data.channelId) return undefined; + if (claims.alertType !== DASHBOARD_AGENT_WATCH_ALERT_TYPE) return undefined; + + return claims; +} + +export async function loader({ request, params }: LoaderFunctionArgs) { + const claims = await authorize(request, params); + // The POST needs the token, and a bare `
` drops search params. + return typedjson({ + valid: claims !== undefined, + formAction: `${new URL(request.url).pathname}${new URL(request.url).search}`, + }); +} + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return typedjson({ success: false as const, message: "Method not allowed" }, { status: 405 }); + } + + const claims = await authorize(request, params); + if (!claims) { + return typedjson( + { + success: false as const, + message: "This link is no longer valid, so we couldn't turn off the alerts.", + }, + { status: 403 } + ); + } + + // Read only for the failure log. The unsubscribe does its own scoped lookup. + const channel = await prisma.projectAlertChannel.findFirst({ + where: { id: claims.channelId }, + select: { projectId: true }, + }); + + try { + const result = await unsubscribeChannelFromWatchAlerts(claims.channelId); + if (!result.ok) { + return result.reason === "conflict" + ? typedjson( + { + success: false as const, + message: "This alert was being changed elsewhere. Please try again.", + }, + { status: 409 } + ) + : typedjson( + { success: false as const, message: "This alert no longer exists." }, + { status: 404 } + ); + } + + return typedjson({ success: true as const, channelName: result.channelName }); + } catch (error) { + logger.error("Failed to turn off watch alerts from an email link", { + error, + channelId: claims.channelId, + projectId: channel?.projectId, + }); + throw error; + } +} + +export default function Page() { + const { valid, formAction } = useTypedLoaderData(); + const result = useTypedActionData(); + const navigation = useNavigation(); + const isLoading = navigation.state !== "idle"; + + if (result?.success) { + return ( + + + {result.channelName} will no longer be alerted when a watch fires. You can turn it back on + from the Alerts page in your project. + + + Dashboard + + + ); + } + + if (!valid || result?.success === false) { + return ( + + + {result?.success === false + ? result.message + : "This link is no longer valid. You can manage alerts from the Alerts page in your project."} + + + Dashboard + + + ); + } + + return ( + + + This stops the alerts this channel receives when a watch you set up with the dashboard agent + fires. Other alerts on the channel are unaffected. + + + + + + ); +} + +function Shell({ title, children }: { title: string; children: React.ReactNode }) { + return ( + + +
+ } + title={title} + /> + {children} +
+ + + ); +} diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts index cdeb6f83a04..c58553ea329 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts @@ -1,16 +1,23 @@ import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { + cancelWatch, chatExists, + countUnreadWatchWakes, + countChatsWithUnreadWork, countUserMessages, createChat, getChatMessages, getSession, + getWatch, listChatIdsWithOpenInvestigations, + listChatIdsWithUnreadWakes, listChats, + markChatRead, + readWatchWakeFeed, renameChat, setChatPinned, - softDeleteChat, } from "@internal/dashboard-agent-db"; +import { watchDraftSchema, type WatchDraft } from "@internal/dashboard-agent-contracts"; import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import type { UIMessage } from "ai"; import { z } from "zod"; @@ -26,6 +33,12 @@ import { $replica } from "~/db.server"; import { env } from "~/env.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { + authorizeWatchEnvironmentById, + deleteChatWithWatches, + listActiveWatchesForChats, + submitDashboardAgentWatch, +} from "~/services/dashboardAgentWatches.server"; import { dashboardAgentApiOrigin, isDashboardAgentConfigured, @@ -61,8 +74,11 @@ const ActionBody = z.object({ "rename", "pin", "delete", + "read", "resolve", "resolve-many", + "watch-cancel", + "watch-create", ]), // Omitted for `create` (the server generates it); required for the rest. chatId: z.string().min(1).optional(), @@ -75,10 +91,17 @@ const ActionBody = z.object({ uri: z.string().optional(), // A JSON array of `trigger://` URIs, for `resolve-many`. uris: z.string().optional(), + // The watch to cancel, for `watch-cancel`. + watchId: z.string().min(1).optional(), + // The configured card, for `watch-create`: a JSON `WatchDraft`. + draft: z.string().optional(), + // Stable per card submission, so a retried `watch-create` repairs instead of repeating. + // Required for `watch-create`: see the check in that branch. + clientRequestId: z.string().min(1).max(64).optional(), }); // History list by default. `?chatId=` returns the stored transcript plus session, -// `?quota=1` the message count. +// `?unread=1` the unread wake count and recent wakes, `?quota=1` the message count. export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const userId = user.id; @@ -97,6 +120,35 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const searchParams = new URL(request.url).searchParams; + // The wake poll runs once a minute per open tab, so it reads only the org id it needs + // and asks the agent DB one question. The list is recent deliveries, not unread ones; + // the client dedupes by id. + if (searchParams.get("unread") === "1") { + const scoped = await $replica.project.findFirst({ + where: { + slug: projectParam, + organization: { slug: organizationSlug, members: { some: { userId } } }, + }, + select: { organizationId: true }, + }); + if (!scoped) return json({ error: "Project not found" }, { status: 404 }); + + const [feed, unreadWork] = await Promise.all([ + readWatchWakeFeed(dashboardAgentDb, { + organizationId: scoped.organizationId, + userId, + deliveredAfter: new Date(Date.now() - 15 * 60 * 1000), + }), + // The dot has two sources; the poll is where a closed panel learns about either. + countChatsWithUnreadWork(dashboardAgentDb, { + organizationId: scoped.organizationId, + userId, + }), + ]); + + return json({ ...feed, unreadWork }); + } + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) return json({ error: "Project not found" }, { status: 404 }); @@ -125,17 +177,45 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { userId, }); - // One query for all the listed chats, never one per row. - const investigatingChatIds = await listChatIdsWithOpenInvestigations(dashboardAgentDb, { - organizationId: project.organizationId, - userId, - }); + // One query each for all the listed chats, never one per row. + const [watchesByChat, unreadWakes, unreadChatIds, investigatingChatIds] = await Promise.all([ + listActiveWatchesForChats({ + chatIds: chats.map((chat) => chat.id), + organizationId: project.organizationId, + userId, + }), + countUnreadWatchWakes(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + listChatIdsWithUnreadWakes(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + listChatIdsWithOpenInvestigations(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + ]); return json({ - chats: chats.map((chat) => ({ - ...chat, - hasOpenInvestigation: investigatingChatIds.has(chat.id), - })), + chats: chats.map((chat) => { + const watches = watchesByChat[chat.id] ?? []; + return { + ...chat, + watches, + hasUnreadWake: unreadChatIds.has(chat.id), + // Work that finished while the chat was closed: the transcript moved on after the + // last time its owner looked. A wake is one way that happens, an answer is another. + hasUnreadWork: + chat.lastMessageAt !== null && + (chat.lastReadAt === null || chat.lastMessageAt > chat.lastReadAt), + // `watches` also carries fired and expired rows, so check for active here. + hasActiveWatch: watches.some((watch) => watch.status === "active"), + hasOpenInvestigation: investigatingChatIds.has(chat.id), + }; + }), + unreadWakes, }); }; @@ -350,6 +430,91 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return json({ resolved }); } + // The configuration card's submit path. The environment comes from the URL and goes + // through the same re-authorization a background tick passes, never from the body. + if (parsed.data.intent === "watch-create") { + // No fallback: a per-condition key would identify the condition rather than this + // submit, so a re-watch could replay a stale terminal outcome. + const clientRequestId = parsed.data.clientRequestId; + if (!clientRequestId) { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + + let draft: WatchDraft; + try { + const result = watchDraftSchema.safeParse(JSON.parse(parsed.data.draft ?? "")); + if (!result.success) { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + draft = result.data; + } catch { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + + const runtimeEnv = await findEnvironmentBySlug(project.id, envParam, userId); + if (!runtimeEnv) return json({ error: "Environment not found" }, { status: 404 }); + + const environment = await authorizeWatchEnvironmentById({ + userId, + environmentId: runtimeEnv.id, + }); + if (!environment) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + + // A watch is chat-bound, so a card submitted from a fresh panel creates a chat. + const targetChatId = parsed.data.chatId; + if ( + targetChatId && + !(await chatExists(dashboardAgentDb, { + chatId: targetChatId, + userId, + organizationId: project.organizationId, + })) + ) { + return json({ error: "Chat not found", code: "chat_not_found" }, { status: 404 }); + } + + // The request record is written before the watch and the confirmation after, so a + // half-finished submit is repairable and never leaves a watch nobody can see. + const result = await submitDashboardAgentWatch({ + environment, + userId, + organizationId: project.organizationId, + chatId: targetChatId, + clientRequestId, + draft, + }); + + if (!result.ok) { + const status = + result.code === "limit_reached" || + result.code === "duplicate" || + result.code === "request_conflict" + ? 409 + : result.code === "invalid_target" || result.code === "chat_not_found" + ? 404 + : result.code === "not_configured" + ? 501 + : 500; + return json( + { + error: result.error, + code: result.code, + ...(result.existingId ? { existingId: result.existingId } : {}), + }, + { status } + ); + } + + return json({ + chatId: result.chatId, + watching: result.watching, + watchId: result.watchId, + messages: result.messages, + }); + } + const { intent, chatId } = parsed.data; if (!chatId) return json({ error: "chatId is required" }, { status: 400 }); @@ -426,9 +591,44 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return json({ ok: true }); } + // The update is owner-scoped, so a chatId the caller doesn't own is a no-op. + case "read": { + await markChatRead(dashboardAgentDb, { + chatId, + userId, + organizationId: project.organizationId, + }); + return json({ ok: true }); + } + case "delete": { - // `softDeleteChat` is owner-scoped but takes no org, so the org scope has to be - // enforced here. + // `deleteChatWithWatches` is owner-scoped but takes no org, so the org scope has + // to be enforced here. + if ( + !(await chatExists(dashboardAgentDb, { + chatId, + userId, + organizationId: project.organizationId, + })) + ) { + return json({ error: "Chat not found" }, { status: 404 }); + } + // The delete and the watch cancellations land in one transaction. + const { cancelledWatches } = await deleteChatWithWatches({ chatId, userId }); + return json({ ok: true, cancelledWatches }); + } + + // Ownership goes through the chat: the watch must belong to the named chat, and + // that chat to this user in this org. + case "watch-cancel": { + const watchId = parsed.data.watchId; + if (!watchId) return json({ error: "watchId is required" }, { status: 400 }); + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch || watch.chatId !== chatId) { + return json({ error: "Watch not found" }, { status: 404 }); + } + if ( !(await chatExists(dashboardAgentDb, { chatId, @@ -438,7 +638,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { ) { return json({ error: "Chat not found" }, { status: 404 }); } - await softDeleteChat(dashboardAgentDb, { chatId, userId }); + + // `cancelWatch` only touches an active row, so an already-resolved watch keeps + // its outcome and this is a no-op. + await cancelWatch(dashboardAgentDb, { id: watchId, reason: "user" }); return json({ ok: true }); } } diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 288a0c27a84..374440dfa0b 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -33,6 +33,9 @@ import { MachineLabelCombo } from "~/components/MachineLabelCombo"; import { MachineTooltipInfo } from "~/components/MachineTooltipInfo"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { InvestigateButton } from "~/components/dashboard-agent/InvestigateButton"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { isFinalRunStatus } from "~/v3/taskStatus"; +import { runWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; import { failedRunPrompt, isFailedRunStatus, @@ -1147,6 +1150,16 @@ function RunBody({ runFriendlyId={run.friendlyId} /> ) : null} + {/* The universal `Watch…` entry (§2.1), pre-filled with this run's + recommendation: tell me when it finishes. Only while the run can + still change — a finished run has nothing left to wait for. */} + {isFinalRunStatus(run.status) ? null : ( + + )} {run.error && ( diff --git a/apps/webapp/app/services/dashboardAgent.server.ts b/apps/webapp/app/services/dashboardAgent.server.ts index 88eec884d5a..9b1168ce2ec 100644 --- a/apps/webapp/app/services/dashboardAgent.server.ts +++ b/apps/webapp/app/services/dashboardAgent.server.ts @@ -13,13 +13,16 @@ const TASK_ID = "dashboard-agent"; // what lets it exchange the token for an env JWT (the gate on the exchange // route); the rest scope the actual reads. No write/admin scopes, so even a // leaked token can't mutate anything. -const DASHBOARD_AGENT_UAT_CAP = [ +export const DASHBOARD_AGENT_UAT_CAP = [ "read:apiKeys", "read:runs", "read:deployments", "read:environments", "read:errors", "read:query", + // Queue metrics ride on `read:query`, but a queue's own row — paused, depth, limit — + // is a `queues` read, and without it the agent can only see the metrics window. + "read:queues", ]; // Minted fresh on every turn (the `in` proxy injects it), so the lifetime only diff --git a/apps/webapp/app/services/dashboardAgentAlertContext.server.ts b/apps/webapp/app/services/dashboardAgentAlertContext.server.ts new file mode 100644 index 00000000000..7266991265f --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentAlertContext.server.ts @@ -0,0 +1,57 @@ +/** + * From the turn's environment scope and a chat id to an authorized environment. Same order + * of authority as the watches route: token environment, chat ownership, re-authorization. + */ + +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + authorizeWatchEnvironmentById, + resolveChatWatchContext, +} from "~/services/dashboardAgentWatches.server"; + +export type AgentAlertContextError = "chat_not_found" | "invalid_target" | "environment_mismatch"; + +export type AgentAlertContext = + | { ok: true; environment: AuthenticatedEnvironment } + | { ok: false; code: AgentAlertContextError; error: string }; + +export async function resolveAgentAlertContext(params: { + userId: string; + chatId: string; + /** The turn's environment scope, off the user-actor token. The authority here. */ + environmentId: string; + /** Optional echoes from the request body. Checked, never trusted. */ + claimedEnvironmentId?: string; + claimedProjectRef?: string; +}): Promise { + if (params.claimedEnvironmentId && params.claimedEnvironmentId !== params.environmentId) { + return { + ok: false, + code: "environment_mismatch", + error: "That environment isn't the one this chat is open in.", + }; + } + + const chat = await resolveChatWatchContext({ chatId: params.chatId, userId: params.userId }); + if (!chat) { + return { ok: false, code: "chat_not_found", error: "Chat not found" }; + } + + const environment = await authorizeWatchEnvironmentById({ + userId: params.userId, + environmentId: params.environmentId, + }); + if (!environment || environment.organizationId !== chat.organizationId) { + return { ok: false, code: "invalid_target", error: "Environment not found" }; + } + + if (params.claimedProjectRef && environment.project.externalRef !== params.claimedProjectRef) { + return { + ok: false, + code: "environment_mismatch", + error: "That project isn't the one this chat is open in.", + }; + } + + return { ok: true, environment }; +} diff --git a/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts b/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts new file mode 100644 index 00000000000..4c43b962f33 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts @@ -0,0 +1,63 @@ +/** + * The credential in a watch alert email's unsubscribe link. HS256 over `SESSION_SECRET` with + * a prefix and `kind` claim disjoint from every other token signed with that secret. + */ + +import { generateJWT, validateJWT } from "@trigger.dev/core/v3/jwt"; +import { env } from "~/env.server"; + +const UNSUBSCRIBE_TOKEN_PREFIX = "tr_daau_"; +const UNSUBSCRIBE_TOKEN_KIND = "dashboard_agent_alert_unsubscribe"; +const UNSUBSCRIBE_PURPOSE = "unsubscribe"; + +/** Long-lived: an alert email has to keep working months after it arrived. */ +const UNSUBSCRIBE_TOKEN_TTL = "365d"; + +export type UnsubscribeTokenClaims = { channelId: string; alertType: string }; + +export async function signDashboardAgentAlertUnsubscribeToken( + secret: string, + opts: { channelId: string; alertType: string } +): Promise { + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: UNSUBSCRIBE_TOKEN_KIND, + purpose: UNSUBSCRIBE_PURPOSE, + sub: opts.channelId, + alertType: opts.alertType, + }, + expirationTime: UNSUBSCRIBE_TOKEN_TTL, + }); + + return `${UNSUBSCRIBE_TOKEN_PREFIX}${jwt}`; +} + +export async function verifyDashboardAgentAlertUnsubscribeToken( + secret: string, + token: string +): Promise { + if (!token.startsWith(UNSUBSCRIBE_TOKEN_PREFIX)) return; + + const result = await validateJWT(token.slice(UNSUBSCRIBE_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== UNSUBSCRIBE_TOKEN_KIND) return; + if (payload.purpose !== UNSUBSCRIBE_PURPOSE) return; + if (typeof payload.sub !== "string" || payload.sub.length === 0) return; + if (typeof payload.alertType !== "string" || payload.alertType.length === 0) return; + + return { channelId: payload.sub, alertType: payload.alertType }; +} + +export function mintDashboardAgentAlertUnsubscribeToken(opts: { + channelId: string; + alertType: string; +}): Promise { + return signDashboardAgentAlertUnsubscribeToken(env.SESSION_SECRET, opts); +} + +export function verifyUnsubscribeToken(token: string): Promise { + return verifyDashboardAgentAlertUnsubscribeToken(env.SESSION_SECRET, token); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts b/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts new file mode 100644 index 00000000000..7cc4e6feb2c --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts @@ -0,0 +1,291 @@ +/** + * The seam between a watch firing and the standard alert pipeline: the enqueue, plus the + * gate both the fan-out and the agent's subscribe endpoint consult. + */ + +import { type Watch } from "@internal/dashboard-agent-db"; +import { type PrismaClientOrTransaction } from "@trigger.dev/database"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { alertsWorker } from "~/v3/alertsWorker.server"; +import { canAccessDashboardAgent } from "~/v3/canAccessDashboardAgent.server"; +import { CreateAlertChannelService } from "~/v3/services/alerts/createAlertChannel.server"; + +/** The alert type a watch fires under. */ +export const DASHBOARD_AGENT_WATCH_ALERT_TYPE = "DASHBOARD_AGENT_WATCH" as const; + +/** What the enqueue needs off a watch row, rather than the full row. */ +export type WatchFiredAlertSource = Pick< + Watch, + | "id" + | "identity" + | "spec" + | "organizationId" + | "projectId" + | "environmentId" + | "userId" + | "firedAt" + | "lastResult" + | "resolution" + | "observedOutcome" +>; + +/** + * Queue the alert fan-out for a resolved watch. Only `fired` dispatches; an expiry is + * narrated in the chat. The job id is the idempotency key: one fan-out per watch. + */ +export async function enqueueWatchFiredAlert( + watch: WatchFiredAlertSource, + outcome: "fired" | "expired" +): Promise { + if (outcome !== "fired") return; + + await alertsWorker.enqueue({ + id: `watch-alert:${watch.id}`, + job: "v3.deliverDashboardAgentWatchAlert", + payload: { + watchId: watch.id, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + userId: watch.userId, + identity: watch.identity, + kind: watch.spec.kind, + note: watch.spec.note, + firedAt: (watch.firedAt ?? new Date()).toISOString(), + facts: watch.lastResult ?? {}, + // The frozen resolved result: the email renders from these and never re-reads + // the source. + resolution: watch.resolution ?? "condition_met", + observed: watch.observedOutcome ?? undefined, + }, + }); +} + +export type DashboardAgentAlertDenyReason = + /** The user can't use the dashboard agent, so its watches can't alert either. */ + | "dashboard_agent_disabled" + /** This installation has no alert email transport configured. */ + | "email_alerts_not_configured"; + +export type DashboardAgentAlertGate = + | { allowed: true } + | { allowed: false; reason: DashboardAgentAlertDenyReason }; + +/** + * May this user's watches alert at all? Operational checks only, no plan check: billing + * gates that separately. `organizationId` stays in the signature for that gate. + */ +export async function canUseDashboardAgentAlerts(params: { + userId: string; + organizationSlug: string; + organizationId: string; + isAdmin?: boolean; + orgFeatureFlags?: Record | null; +}): Promise { + const hasAgent = await canAccessDashboardAgent({ + userId: params.userId, + isAdmin: params.isAdmin ?? false, + // Never an impersonated session: this runs in the background. + isImpersonating: false, + organizationSlug: params.organizationSlug, + orgFeatureFlags: params.orgFeatureFlags, + }); + if (!hasAgent) return { allowed: false, reason: "dashboard_agent_disabled" }; + + return { allowed: true }; +} + +/** + * Whether a fired watch in this environment would already reach this user outside the + * chat. Advisory only: the watch already exists, so every failure answers `none`. + */ +export async function resolveWatchEmailAlertsState(params: { + userId: string; + environment: AuthenticatedEnvironment; +}): Promise<"subscribed" | "none" | "unavailable"> { + const { userId, environment } = params; + try { + // Another member's channel mails them, not this user, so only this user's own channel + // answers "subscribed". + const owner = await resolveWatchAlertOwnership(userId, $replica); + const channel = owner + ? await $replica.projectAlertChannel.findFirst({ + where: { + projectId: environment.project.id, + deduplicationKey: owner.deduplicationKey, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + environmentTypes: { has: environment.type }, + }, + select: { id: true }, + }) + : null; + if (channel) return "subscribed"; + + const gate = await canUseDashboardAgentAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + orgFeatureFlags: environment.organization.featureFlags as Record | null, + }); + return gate.allowed ? "none" : "unavailable"; + } catch (error) { + logger.error("Failed to resolve dashboard agent watch alert state", { + error, + userId, + organizationId: environment.organizationId, + projectId: environment.project.id, + environmentId: environment.id, + }); + return "none"; + } +} + +/** The same gate plus an email transport, or the channel would never deliver. */ +export async function canUseDashboardAgentEmailAlerts( + params: Parameters[0] & { projectId: string } +): Promise { + const base = await canUseDashboardAgentAlerts(params); + if (!base.allowed) return base; + + // Mirrors what the alerts email client needs, not resend specifically. + if (env.ALERT_FROM_EMAIL === undefined || env.ALERT_EMAIL_TRANSPORT === undefined) { + return { allowed: false, reason: "email_alerts_not_configured" }; + } + + return { allowed: true }; +} + +// A channel has no owner column, so this key is the only record of whose channel it is. +export function watchAlertDeduplicationKey(email: string): string { + return `dashboard-agent-watch:${email}`; +} + +/** + * The one place a user id becomes watch-alert ownership. Reading state, subscribing and + * unsubscribing all go through here, so they cannot disagree about whose channel is whose. + */ +async function resolveWatchAlertOwnership( + userId: string, + db: PrismaClientOrTransaction = prisma +): Promise<{ email: string; deduplicationKey: string } | undefined> { + const user = await db.user.findFirst({ where: { id: userId }, select: { email: true } }); + if (!user) return undefined; + return { email: user.email, deduplicationKey: watchAlertDeduplicationKey(user.email) }; +} + +export type SubscribeToWatchAlertsResult = + | { ok: true; email: string } + | { ok: false; reason: DashboardAgentAlertDenyReason | "user_not_found" }; + +/** + * Subscribe the signed-in user's own account email to this project's watch alerts. The + * address is never taken from the request, and the dedup key is stable per (email, project). + */ +export async function subscribeUserToWatchAlerts(params: { + userId: string; + environment: { + type: string; + organizationId: string; + organization: { slug: string }; + project: { id: string; externalRef: string }; + }; +}): Promise { + const { userId, environment } = params; + + const gate = await canUseDashboardAgentEmailAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + projectId: environment.project.id, + }); + if (!gate.allowed) return { ok: false, reason: gate.reason }; + + const owner = await resolveWatchAlertOwnership(userId); + if (!owner) return { ok: false, reason: "user_not_found" }; + + const service = new CreateAlertChannelService(); + await service.call(environment.project.externalRef, userId, { + name: `Watch alerts for ${owner.email}`, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE], + environmentTypes: [environment.type as never], + deduplicationKey: owner.deduplicationKey, + channel: { type: "EMAIL", email: owner.email }, + }); + + return { ok: true, email: owner.email }; +} + +export type UnsubscribeResult = + | { ok: true; channelName: string; disabledChannel: boolean } + | { ok: false; reason: "not_found" | "conflict" }; + +/** How many times a lost race is retried before the caller is told to try again. */ +const UNSUBSCRIBE_ATTEMPTS = 3; + +/** + * Take `DASHBOARD_AGENT_WATCH` off a channel, disabling one left with no alert types. The + * write is conditional on the list the read saw, so a concurrent edit fails this attempt. + * + * A project is shared by every member, so a request-driven caller must pass + * `organizationId` and `ownerUserId` too. + */ +export async function unsubscribeChannelFromWatchAlerts( + channelId: string, + options: { projectId?: string; organizationId?: string; ownerUserId?: string } = {}, + db: PrismaClientOrTransaction = prisma +): Promise { + let ownerKey: string | undefined; + if (options.ownerUserId) { + const owner = await resolveWatchAlertOwnership(options.ownerUserId, db); + if (!owner) return { ok: false, reason: "not_found" }; + ownerKey = owner.deduplicationKey; + } + + const scope = { + id: channelId, + ...(options.projectId ? { projectId: options.projectId } : {}), + ...(options.organizationId ? { project: { organizationId: options.organizationId } } : {}), + ...(ownerKey ? { deduplicationKey: ownerKey } : {}), + }; + + for (let attempt = 0; attempt < UNSUBSCRIBE_ATTEMPTS; attempt++) { + const channel = await db.projectAlertChannel.findFirst({ + where: scope, + select: { name: true, alertTypes: true, projectId: true, deduplicationKey: true }, + }); + // A channel this alert type was never on is out of scope: stripping nothing off it + // would still report success, and an empty list would disable it. + if (!channel || !channel.alertTypes.includes(DASHBOARD_AGENT_WATCH_ALERT_TYPE)) { + return { ok: false, reason: "not_found" }; + } + + const remaining = channel.alertTypes.filter( + (type) => type !== DASHBOARD_AGENT_WATCH_ALERT_TYPE + ); + + const { count } = await db.projectAlertChannel.updateMany({ + // Compare-and-swap on the row the scoped read returned. `updateMany` takes no relation + // filter, so the org scope is carried by the read's `projectId`. + where: { + id: channelId, + projectId: channel.projectId, + deduplicationKey: channel.deduplicationKey, + alertTypes: { equals: channel.alertTypes }, + }, + data: { + alertTypes: remaining, + ...(remaining.length === 0 ? { enabled: false } : {}), + }, + }); + + if (count > 0) { + return { ok: true, channelName: channel.name, disabledChannel: remaining.length === 0 }; + } + } + + return { ok: false, reason: "conflict" }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts b/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts new file mode 100644 index 00000000000..6f7b071d12b --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts @@ -0,0 +1,304 @@ +/** + * The batch check: every due watch of one (environment, cadence) group in one pass. Each row is + * the authority on its own snapshot, and each initiating user is re-authorized before any read. + */ + +import { + cancelWatch, + claimWatchBatchTick, + listActiveWatchesForBatch, + listWatchesAwaitingDeliveryForBatch, + recordWatchAttempt, + recordWatchCheck, + stopWatchBatch, + WATCH_DELIVERY_CLAIM_STALE_MS, + type Watch, +} from "@internal/dashboard-agent-db"; +import type { WatchBatchCheckEntry, WatchBatchCheckResponse } from "@internal/dashboard-agent"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { + checkWatch, + previousCheckFacts, + type WatchCheckDeps, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { + authorizeWatchEnvironment, + type WatchAuthorization, +} from "~/services/dashboardAgentWatches.server"; +import { + mintDashboardAgentWatchToken, + WATCH_TOKEN_GRACE_MS, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * How early a watch may be checked and still count as due, so a tick landing seconds + * early doesn't defer it a whole cadence. Capped at half a cadence. + */ +function dueSlackMs(cadenceMinutes: number): number { + return Math.min(30_000, (cadenceMinutes * 60_000) / 2); +} + +/** Small on purpose: it stops one slow condition serializing the group, not to fan out. */ +const EVALUATION_CONCURRENCY = 8; + +export type WatchBatchCheckDeps = { + now?: () => Date; + /** The group's active watches. */ + listActive?: (params: { environmentId: string; cadenceMinutes: number }) => Promise; + /** The group's resolved watches whose wake is still owed. */ + listOwed?: (params: { + environmentId: string; + cadenceMinutes: number; + claimStaleBefore: Date; + }) => Promise; + /** Re-authorization of one watch's initiating user. */ + authorize?: (watch: Watch) => Promise; + /** The environment readers the conditions run against. */ + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + /** The per-watch token the fired / investigate callbacks are made with. */ + mintToken?: (watch: Watch) => Promise; + concurrency?: number; +}; + +/** + * Run one batch tick's checks. The claim decides whether this run owns the tick and keeps the + * schedule single-file; the guarded transition and fenced delivery claim stop a double fire. + */ +export async function runWatchBatchCheck( + params: { environmentId: string; cadenceMinutes: number; epoch: number; tick: number }, + deps: WatchBatchCheckDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const listActive = + deps.listActive ?? ((args) => listActiveWatchesForBatch(dashboardAgentDb, args)); + const listOwed = + deps.listOwed ?? ((args) => listWatchesAwaitingDeliveryForBatch(dashboardAgentDb, args)); + const mintToken = + deps.mintToken ?? + ((watch: Watch) => + mintDashboardAgentWatchToken({ watchId: watch.id, expiresAt: watch.expiresAt })); + + const claimed = await claimWatchBatchTick(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: params.epoch, + // The tick a run carries is the generation it owns. + generation: params.tick, + }); + if (!claimed) { + logger.debug("Dashboard agent watch batch: the tick is stale", params); + return { stale: true }; + } + + const active = await listActive({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + + const due = active.filter((watch) => isDue(watch, params.cadenceMinutes, now)); + const evaluated = await evaluateGroup(due, params, { ...deps, now: () => now }, mintToken); + + // Wakes this group still owes. Read after the evaluation, so a wake this tick resolved + // and failed to deliver is already in it. + const owed = await listOwed({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + claimStaleBefore: new Date(now.getTime() - WATCH_DELIVERY_CLAIM_STALE_MS), + }); + + const deliveries = await Promise.all( + owed.map(async (watch) => ({ + watchId: watch.id, + token: await mintToken(watch), + // A delivery decides nothing, so it claims no generation. + tick: 0, + deliverOnly: true as const, + })) + ); + + // The chain only stops with nothing to poll and nothing owed: stopping while a wake is + // owed strands it. Fenced on the epoch, so it can only end this run's own chain. + const continues = active.length > 0 || owed.length > 0; + if (!continues) { + await stopWatchBatch(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: params.epoch, + }); + } + + return { watches: [...evaluated, ...deliveries], continues }; +} + +/** + * A watch whose window closes before the next tick is due now, so its final evaluation is + * never missed. A watch past the token grace is never due: the expiry sweep owns it. + */ +export function isDue(watch: Watch, cadenceMinutes: number, now: Date): boolean { + const nowMs = now.getTime(); + const cadenceMs = cadenceMinutes * 60_000; + + if (nowMs > watch.expiresAt.getTime() + WATCH_TOKEN_GRACE_MS) return false; + if (watch.expiresAt.getTime() <= nowMs + cadenceMs) return true; + + const lastChecked = watch.lastCheckedAt?.getTime(); + return lastChecked === undefined || lastChecked <= nowMs - cadenceMs + dueSlackMs(cadenceMinutes); +} + +/** + * Evaluate the due watches against one set of readers. Authorization is cached per (user, org, + * project); each watch runs in its own try, so a failure is that watch's answer alone. + */ +async function evaluateGroup( + due: Watch[], + params: { environmentId: string; cadenceMinutes: number }, + deps: WatchBatchCheckDeps, + mintToken: (watch: Watch) => Promise +): Promise { + if (due.length === 0) return []; + + const now = deps.now?.() ?? new Date(); + const authorize = deps.authorize ?? defaultAuthorize; + const buildCheckDeps = deps.checkDeps ?? watchCheckDeps; + + const authorizations = new Map>(); + const authorizeOnce = (watch: Watch) => { + const key = `${watch.userId}:${watch.organizationId}:${watch.projectId}`; + const cached = authorizations.get(key); + if (cached) return cached; + const pending = authorize(watch); + authorizations.set(key, pending); + return pending; + }; + + // Built from the first authorization that passes, then shared: every row in the group + // names the same environment. + let readers: WatchCheckDeps | undefined; + + const evaluateOne = async ( + watch: Watch, + base: { watchId: string; token: string; tick: number } + ): Promise => { + const authorization = await authorizeOnce(watch); + if (!authorization.ok) { + // Cancel before anything is read: a watch must not outlive its creator's access. + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "access_revoked" }); + return { ...base, code: "access_revoked", error: "Access to this environment was revoked" }; + } + + readers ??= shareReads(buildCheckDeps(authorization.environment, now)); + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const final = watch.expiresAt.getTime() <= now.getTime(); + const outcome = await checkWatch( + watch.spec, + readers, + // A check that couldn't read anything freezes a streak instead of resetting it. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch batch: a check failed", { + error, + watchId: watch.id, + environmentId: params.environmentId, + }) + ); + + // Only a real evaluation is recorded, final or not: `unavailable` means nothing was read, + // so writing it would move `lastCheckedAt` and overwrite the facts a streak lives in. + // Guarded on `active`, and never touches `tickCount`. + if (outcome.result !== "unavailable") { + await recordWatchCheck(dashboardAgentDb, { + id: watch.id, + lastResult: { + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + final, + }, + }); + } else { + // Looked at, not checked: this rotates the watch out of its group's head without + // touching its dueness or the facts its streak lives in. + await recordWatchAttempt(dashboardAgentDb, { id: watch.id }); + } + + return { ...base, result: outcome.result, facts: outcome.facts, observed: outcome.observed }; + }; + + return mapWithConcurrency(due, deps.concurrency ?? EVALUATION_CONCURRENCY, async (watch) => { + // Minted outside the try, because the catch below needs a token it can't fail to have. + const base = { watchId: watch.id, token: await mintToken(watch), tick: watch.tickCount + 1 }; + try { + return await evaluateOne(watch, base); + } catch (error) { + logger.error("Dashboard agent watch batch: a watch couldn't be evaluated", { + watchId: watch.id, + environmentId: params.environmentId, + error, + }); + // `unavailable` is never read as true or false: the watch keeps its state. Still a + // look, so the fairness key moves even when nothing else does. + await recordWatchAttempt(dashboardAgentDb, { id: watch.id }).catch(() => {}); + return { ...base, result: "unavailable" as const, error: (error as Error).message }; + } + }); +} + +/** + * Wrap a batch's readers so each distinct read happens once. `now` is fixed for the batch, so + * a reader's answer is a pure function of its arguments. Failed reads are cached too. + * + * Exported for the expiry sweep, which finalizes the same rows against the same readers. + */ +export function shareReads(readers: WatchCheckDeps): WatchCheckDeps { + const cache = new Map>(); + const once = (name: string, read: (...args: A) => Promise) => { + return (...args: A): Promise => { + const key = `${name}:${JSON.stringify(args)}`; + const cached = cache.get(key); + if (cached) return cached as Promise; + const pending = read(...args); + cache.set(key, pending); + return pending; + }; + }; + + return { + readRun: once("readRun", readers.readRun), + queueExists: once("queueExists", readers.queueExists), + readQueueDepth: once("readQueueDepth", readers.readQueueDepth), + readQueueOldestAge: once("readQueueOldestAge", readers.readQueueOldestAge), + readErrorRecurrence: once("readErrorRecurrence", readers.readErrorRecurrence), + readHealth: once("readHealth", readers.readHealth), + }; +} + +/** `mapper` over `items`, at most `limit` in flight. Order is preserved. */ +export async function mapWithConcurrency( + items: T[], + limit: number, + mapper: (item: T) => Promise +): Promise { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(Math.max(limit, 1), items.length) }, async () => { + while (next < items.length) { + const index = next++; + results[index] = await mapper(items[index]!); + } + }); + await Promise.all(workers); + return results; +} + +function defaultAuthorize(watch: Watch): Promise { + return authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts b/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts new file mode 100644 index 00000000000..fc41bcba038 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts @@ -0,0 +1,106 @@ +/** + * The reader contract every watch condition family shares, plus the duration formatting + * they all label with. No IO of its own. + */ + +import { formatDurationMilliseconds } from "@trigger.dev/core/v3/utils/durations"; +import type { WatchCheckResult, WatchObservedOutcome } from "@internal/dashboard-agent-contracts"; + +/** The single run point-read. Postgres is authoritative for run state. */ +export type WatchRunRow = { + friendlyId: string; + status: string; + queue: string; + createdAt: Date; + /** Stamped when the run entered the queue. NULL while a run is delayed. */ + queuedAt: Date | null; + /** Set once the run is dequeued. */ + startedAt: Date | null; + completedAt: Date | null; + delayUntil: Date | null; +}; + +export type WatchQueueDepth = { + /** Pending count for the queue, as of `asOf`. */ + depth: number; + source: "live_queue" | "queue_metrics"; + /** A stale reading can never answer "drained". */ + current: boolean; + /** What instant the reading describes, when it isn't the live counter. */ + asOf?: Date; +}; + +/** + * The oldest still-waiting run's age in one queue. A non-current age is wrong in both + * directions, so `checkQueueOldestAge` refuses it rather than comparing it. + */ +export type WatchQueueOldestAge = { + /** Age of the oldest run still waiting, in ms. Null when nothing is waiting. */ + ageMs: number | null; + source: "live_queue" | "queue_metrics"; + current: boolean; + asOf?: Date; +}; + +/** What we know about the watched error's occurrences relative to `since`. */ +export type WatchErrorRecurrence = { + /** Earliest occurrence proven after `since`. Null with a `lastSeenAt` means not since. */ + occurredAt: Date | null; + /** How precisely `occurredAt` is known: to the millisecond, or to its minute. */ + occurredAtPrecision: "exact" | "minute" | null; + /** Occurrences after `since`. A lower bound when `countApproximate`. */ + countSince: number; + /** True when occurrences in the watch's creation minute can't be separated out. */ + countApproximate: boolean; + /** The fingerprint's most recent occurrence, whenever it was. */ + lastSeenAt: Date | null; +}; + +export type WatchHealthSeverity = "ok" | "warn" | "crit"; + +export type WatchHealthSnapshot = { + /** `facts.trustworthy` from the health report. Untrustworthy never fires recovery. */ + trustworthy: boolean; + severity: WatchHealthSeverity; +}; + +/** + * The readers a check may use. Each may throw, which the caller turns into `unavailable`. + * `null` means the source answered and there is nothing there. + */ +export type WatchCheckDeps = { + /** Run point-read by public run id, scoped to the watch's environment. */ + readRun: (runId: string) => Promise; + /** Does this queue exist in the watch's environment? */ + queueExists: (queue: string) => Promise; + /** Current pending count, live run-queue first with a ClickHouse fallback. */ + readQueueDepth: (queue: string) => Promise; + /** Age of the oldest run still waiting in the queue, right now. */ + readQueueOldestAge: (queue: string) => Promise; + /** `null` means the fingerprint has no occurrences at all in this environment. */ + readErrorRecurrence: (fingerprint: string, since: Date) => Promise; + /** The health report's current verdict for the watch's environment. */ + readHealth: () => Promise; +}; + +export type WatchCheckInput = { + now: Date; + /** The recurrence window's start: the server-set `spec.since`, never caller-set. */ + since: Date; + /** + * The previous check's facts, for the stateful kinds. A check's own facts are the only + * storage for its state. Absent means no prior observation, never zero. + */ + previous?: Record | null; +}; + +export type WatchCheckOutcome = { + result: WatchCheckResult; + facts: Record; + /** Frozen onto the row by the resolving transition, so no surface re-reads the source. */ + observed: WatchObservedOutcome; +}; + +export function formatMs(ms: number): string { + return formatDurationMilliseconds(ms, { style: "short", maxDecimalPoints: 0 }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts b/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts new file mode 100644 index 00000000000..ac49aaacf68 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts @@ -0,0 +1,337 @@ +/** + * Default IO wiring for the watch checks. Run state and queue existence are authoritative + * Postgres point-reads, and readers throw rather than invent a zero on a broken source. + */ + +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { $replica, prisma } from "~/db.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { ReportPresenter } from "~/presenters/v3/reports/ReportPresenter.server"; +import { engine } from "~/v3/runEngine.server"; +import { runStore } from "~/v3/runStore.server"; +import type { + WatchCheckDeps, + WatchErrorRecurrence, + WatchHealthSeverity, + WatchHealthSnapshot, + WatchQueueDepth, + WatchQueueOldestAge, + WatchRunRow, +} from "./dashboardAgentWatchChecks"; + +const WATCH_RUN_SELECT = { + friendlyId: true, + status: true, + queue: true, + createdAt: true, + queuedAt: true, + startedAt: true, + completedAt: true, + delayUntil: true, +} as const; + +/** The single Postgres point-read: one run, scoped to the watch's environment. */ +export async function readWatchRun( + runFriendlyId: string, + environmentId: string +): Promise { + const run = await runStore.findRun( + { friendlyId: runFriendlyId, runtimeEnvironmentId: environmentId }, + { select: WATCH_RUN_SELECT }, + $replica + ); + return run ?? null; +} + +/** One Postgres point-read: does this queue exist in the environment? */ +export async function watchQueueExists(environmentId: string, queueName: string): Promise { + const queue = await $replica.taskQueue.findFirst({ + where: { runtimeEnvironmentId: environmentId, name: queueName }, + select: { id: true }, + }); + return queue !== null; +} + +/** The same run read on the primary, for a target that may have been created a moment ago. */ +export async function readWatchRunOnPrimary( + runFriendlyId: string, + environmentId: string +): Promise { + const run = await runStore.findRunOnPrimary( + { friendlyId: runFriendlyId, runtimeEnvironmentId: environmentId }, + { select: WATCH_RUN_SELECT } + ); + return run ?? null; +} + +/** The same queue read on the primary. */ +export async function watchQueueExistsOnPrimary( + environmentId: string, + queueName: string +): Promise { + const queue = await prisma.taskQueue.findFirst({ + where: { runtimeEnvironmentId: environmentId, name: queueName }, + select: { id: true }, + }); + return queue !== null; +} + +/** How far back the ClickHouse depth fallback looks when the live counter is down. */ +const DEPTH_FALLBACK_MINUTES = 10; +const DEPTH_FALLBACK_BUCKET_SECONDS = 60; +/** + * How far behind `now` the newest analytics bucket may end and still count as current. + * One bucket of slack: anything older leaves runs queued in the gap invisible. + */ +const DEPTH_FRESH_TOLERANCE_MS = DEPTH_FALLBACK_BUCKET_SECONDS * 1000; + +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().slice(0, 19).replace("T", " "); +} + +/** ClickHouse renders DateTime without a zone; the column is UTC. */ +function parseClickhouseDateTime(value: string): Date { + return new Date(`${value.replace(" ", "T")}Z`); +} + +/** + * Current pending count for one queue. The live counter is the truth; the ClickHouse fallback + * reports the newest bucket's peak depth and is `current` only if it reaches the present. + */ +export async function readWatchQueueDepth( + environment: AuthenticatedEnvironment, + queueName: string, + now: Date = new Date() +): Promise { + const live = await engine.lengthOfQueue(environment, queueName).catch(() => null); + if (typeof live === "number" && Number.isFinite(live)) { + return { depth: live, source: "live_queue", current: true, asOf: now }; + } + + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + environment.organizationId, + "query" + ); + + const bucketMs = DEPTH_FALLBACK_BUCKET_SECONDS * 1000; + const endMs = Math.ceil(now.getTime() / bucketMs) * bucketMs; + const startMs = endMs - DEPTH_FALLBACK_MINUTES * 60_000; + + const [error, rows] = await clickhouse.queueMetrics.depthSparklines({ + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: environment.id, + queueNames: [queueName], + startTime: formatClickhouseDateTime(new Date(startMs)), + endTime: formatClickhouseDateTime(new Date(endMs)), + bucketSeconds: DEPTH_FALLBACK_BUCKET_SECONDS, + }); + + if (error) throw error; + if (!rows || rows.length === 0) return null; + + // Newest bucket wins: the closest the rollup gets to now. + const newest = rows.reduce((best, row) => (row.bucket > best.bucket ? row : best), rows[0]!); + const bucketEnd = new Date(parseClickhouseDateTime(newest.bucket).getTime() + bucketMs); + const current = bucketEnd.getTime() >= now.getTime() - DEPTH_FRESH_TOLERANCE_MS; + + return { depth: newest.depth, source: "queue_metrics", current, asOf: bucketEnd }; +} + +/** + * How long the oldest still-waiting run in a queue has waited: for a concurrency-keyed queue the + * worst across keys with a live backlog. `null` can't be read, `ageMs: null` is empty. + */ +export async function readWatchQueueOldestAge( + environment: AuthenticatedEnvironment, + queueName: string, + now: Date = new Date() +): Promise { + const [breakdown, oldestQueuedAt] = await Promise.all([ + engine + .concurrencyKeyBreakdown(environment, queueName, { limit: OLDEST_AGE_CK_LIMIT }) + .catch(() => null), + engine.oldestMessageInQueue(environment, queueName).catch(() => null), + ]); + + // A partial read would under-report the wait and silently miss the SLA, so either read + // failing makes the whole reading unavailable rather than a healthy zero. + if (breakdown === null || oldestQueuedAt === null) return null; + + const waitingKeys = breakdown.keys.filter((key) => key.queued > 0); + const ageMs = + waitingKeys.length > 0 + ? waitingKeys.reduce((max, key) => Math.max(max, now.getTime() - key.oldestEnqueuedAt), 0) + : typeof oldestQueuedAt === "number" + ? Math.max(0, now.getTime() - oldestQueuedAt) + : null; + + return { ageMs, source: "live_queue", current: true, asOf: now }; +} + +/** Same cap the queue detail page reads keys with. */ +const OLDEST_AGE_CK_LIMIT = 50; + +const MINUTE_MS = 60_000; + +type OrganizationClickhouse = Awaited< + ReturnType +>; + +/** + * The fingerprint's most recent occurrence at millisecond precision, from `errors_v1`. The + * per-minute rollup can't separate the prompting error from a recurrence in the same minute. + */ +async function readErrorLastSeen( + clickhouse: OrganizationClickhouse, + environment: AuthenticatedEnvironment, + fingerprint: string +): Promise { + const builder = clickhouse.errors.activeErrorsSinceQueryBuilder(); + builder.where("organization_id = {organizationId: String}", { + organizationId: environment.organizationId, + }); + builder.where("project_id = {projectId: String}", { projectId: environment.projectId }); + builder.where("environment_id = {environmentId: String}", { environmentId: environment.id }); + builder.where("error_fingerprint = {fingerprint: String}", { fingerprint }); + builder.groupBy("environment_id, task_identifier, error_fingerprint"); + + const [error, rows] = await builder.execute(); + if (error) throw error; + if (!rows || rows.length === 0) return null; + + let lastSeenMs = 0; + for (const row of rows) { + const ms = Number(row.last_seen); + if (Number.isFinite(ms) && ms > lastSeenMs) lastSeenMs = ms; + } + + return lastSeenMs > 0 ? new Date(lastSeenMs) : null; +} + +/** + * What we know about a fingerprint relative to `since`. `errors_v1` decides whether it + * recurred; the rollup's count is a lower bound when the creation-minute bucket has hits. + */ +export async function readWatchErrorRecurrence( + environment: AuthenticatedEnvironment, + fingerprint: string, + since: Date +): Promise { + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + environment.organizationId, + "logs" + ); + + const lastSeenAt = await readErrorLastSeen(clickhouse, environment, fingerprint); + // Never seen in this environment at all. + if (!lastSeenAt) return null; + + const notRecurred: WatchErrorRecurrence = { + occurredAt: null, + occurredAtPrecision: null, + countSince: 0, + countApproximate: false, + lastSeenAt, + }; + if (lastSeenAt.getTime() <= since.getTime()) return notRecurred; + + // Something landed after `since`; the rollup fills in the count and the minute. + const sinceMinuteMs = Math.floor(since.getTime() / MINUTE_MS) * MINUTE_MS; + const queryBuilder = clickhouse.errors.createOccurrencesQueryBuilder("INTERVAL 1 MINUTE"); + queryBuilder.where("organization_id = {organizationId: String}", { + organizationId: environment.organizationId, + }); + queryBuilder.where("project_id = {projectId: String}", { projectId: environment.projectId }); + queryBuilder.where("environment_id = {environmentId: String}", { environmentId: environment.id }); + queryBuilder.where("error_fingerprint = {fingerprint: String}", { fingerprint }); + // The creation minute is included; its occurrences are counted separately below. + queryBuilder.where("minute >= toStartOfMinute(fromUnixTimestamp64Milli({sinceMs: Int64}))", { + sinceMs: since.getTime(), + }); + queryBuilder.groupBy("error_fingerprint, bucket_epoch"); + queryBuilder.orderBy("bucket_epoch ASC"); + + const [error, rows] = await queryBuilder.execute(); + if (error) throw error; + + let earliestAfterMs: number | null = null; + let countAfter = 0; + let creationMinuteCount = 0; + + for (const row of rows ?? []) { + const bucketMs = row.bucket_epoch * 1000; + if (bucketMs <= sinceMinuteMs) { + creationMinuteCount += row.count; + continue; + } + countAfter += row.count; + if (earliestAfterMs === null || bucketMs < earliestAfterMs) earliestAfterMs = bucketMs; + } + + // The earliest provable occurrence: a bucket starting after the creation minute, or the + // exact `last_seen` when that is the only evidence. + const useBucket = earliestAfterMs !== null && earliestAfterMs < lastSeenAt.getTime(); + + return { + occurredAt: useBucket ? new Date(earliestAfterMs!) : lastSeenAt, + occurredAtPrecision: useBucket ? "minute" : "exact", + // At least the one `errors_v1` proved, even if the rollup lags behind it. + countSince: Math.max(1, countAfter), + countApproximate: creationMinuteCount > 0, + lastSeenAt, + }; +} + +const HEALTH_SEVERITIES = new Set(["ok", "warn", "crit"]); + +/** + * The health report's current verdict, from the existing interpreter. No health reasoning + * is re-implemented here. + */ +export async function readWatchHealth( + environment: AuthenticatedEnvironment +): Promise { + const report = await new ReportPresenter().call({ environment, key: "health" }); + if (!report) return null; + + const severity = report.summary.severity; + if (!HEALTH_SEVERITIES.has(severity)) return null; + + const trustworthy = (report.facts as { trustworthy?: unknown } | undefined)?.trustworthy; + return { + // An absent trust marker counts as untrustworthy. + trustworthy: trustworthy === true, + severity: severity as WatchHealthSeverity, + }; +} + +export function watchCheckDeps( + environment: AuthenticatedEnvironment, + now: Date = new Date() +): WatchCheckDeps { + return { + readRun: (runId) => readWatchRun(runId, environment.id), + queueExists: (queue) => watchQueueExists(environment.id, queue), + readQueueDepth: (queue) => readWatchQueueDepth(environment, queue, now), + readQueueOldestAge: (queue) => readWatchQueueOldestAge(environment, queue, now), + readErrorRecurrence: (fingerprint, since) => + readWatchErrorRecurrence(environment, fingerprint, since), + readHealth: () => readWatchHealth(environment), + }; +} + +/** + * Creation-time deps. The target reads go to the primary, so a run or queue created moments + * ago is visible instead of failing as a non-existent target inside the replication window. + */ +export function watchCreationCheckDeps( + environment: AuthenticatedEnvironment, + now: Date = new Date() +): WatchCheckDeps { + return { + ...watchCheckDeps(environment, now), + readRun: (runId) => readWatchRunOnPrimary(runId, environment.id), + queueExists: (queue) => watchQueueExistsOnPrimary(environment.id, queue), + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchChecks.ts b/apps/webapp/app/services/dashboardAgentWatchChecks.ts new file mode 100644 index 00000000000..717d130ef20 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchChecks.ts @@ -0,0 +1,165 @@ +/** + * Deterministic evaluation of one watch condition, with IO behind `WatchCheckDeps`. `unavailable` + * is never a verdict, durations carry their basis, and `observed` is kept apart from the result. + * + * One module per condition family; this one dispatches and owns the failure envelope. + */ + +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; +import { checkRunFailed, checkRunFinished, checkRunStart } from "./dashboardAgentWatchRunChecks"; +import { + checkBacklogDrain, + checkQueueDepthAbove, + checkQueueDepthBelow, + checkQueueOldestAge, + checkQueueStalled, +} from "./dashboardAgentWatchQueueChecks"; +import { checkErrorRecurrence } from "./dashboardAgentWatchErrorChecks"; +import { checkHealthRecovery } from "./dashboardAgentWatchHealthChecks"; + +export type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, + WatchErrorRecurrence, + WatchHealthSeverity, + WatchHealthSnapshot, + WatchQueueDepth, + WatchQueueOldestAge, + WatchRunRow, +} from "./dashboardAgentWatchCheckBase"; +export { + checkRunFailed, + checkRunFinished, + checkRunStart, + describeRunWait, + isTerminalRunStatus, + type WatchWaitBasis, +} from "./dashboardAgentWatchRunChecks"; +export { + checkBacklogDrain, + checkQueueDepthAbove, + checkQueueDepthBelow, + checkQueueOldestAge, + checkQueueStalled, +} from "./dashboardAgentWatchQueueChecks"; +export { checkErrorRecurrence, normalizeErrorFingerprint } from "./dashboardAgentWatchErrorChecks"; +export { checkHealthRecovery } from "./dashboardAgentWatchHealthChecks"; + +/** + * The previous check's facts out of `lastResult`, which holds raw facts, the check endpoint's + * envelope, or the failure wrapper. The wrapper is unwrapped, so a streak survives a gap. + */ +export function previousCheckFacts(lastResult: unknown): Record | null { + if (!lastResult || typeof lastResult !== "object" || Array.isArray(lastResult)) return null; + const record = lastResult as Record; + + if (record.checkFailed === true) return previousCheckFacts(record.previous); + if (record.facts && typeof record.facts === "object" && !Array.isArray(record.facts)) { + return record.facts as Record; + } + return record; +} + +/** The single place a check failure becomes `unavailable`, never a verdict. */ +export async function checkWatch( + spec: WatchSpec, + deps: WatchCheckDeps, + input: WatchCheckInput, + onError?: (error: unknown) => void +): Promise { + try { + switch (spec.kind) { + case "run_start": + return await checkRunStart(spec, deps, input); + case "run_finished": + return await checkRunFinished(spec, deps, input); + case "run_failed": + return await checkRunFailed(spec, deps, input); + case "backlog_drain": + return await checkBacklogDrain(spec, deps, input); + case "queue_depth_above": + return await checkQueueDepthAbove(spec, deps, input); + case "queue_depth_below": + return await checkQueueDepthBelow(spec, deps, input); + case "queue_stalled": + return await checkQueueStalled(spec, deps, input); + case "queue_oldest_age": + return await checkQueueOldestAge(spec, deps, input); + case "error_recurrence": + return await checkErrorRecurrence(spec, deps, input); + case "health_recovery": + return await checkHealthRecovery(spec, deps, input); + default: { + const unreachable: never = spec; + throw new Error(`Unhandled watch kind: ${JSON.stringify(unreachable)}`); + } + } + } catch (error) { + onError?.(error); + return { + result: "unavailable", + facts: { kind: spec.kind, reason: "check_failed" }, + observed: unobservedOutcome(spec), + }; + } +} + +/** + * The observation for a check that couldn't run. `verified: false` means the condition + * couldn't be confirmed, not that it didn't happen. + */ +export function unobservedOutcome(spec: WatchSpec): WatchObservedOutcome { + switch (spec.kind) { + case "run_start": + return { kind: "run_start", verified: false, status: null, started: false }; + case "run_finished": + return { kind: "run_finished", verified: false, finalStatus: null, durationMs: null }; + case "run_failed": + return { kind: "run_failed", verified: false, finalStatus: null, durationMs: null }; + case "backlog_drain": + return { kind: "backlog_drain", verified: false, depth: null }; + case "queue_depth_above": + return { + kind: "queue_depth_above", + verified: false, + depth: null, + threshold: spec.threshold, + }; + case "queue_depth_below": + return { + kind: "queue_depth_below", + verified: false, + depth: null, + threshold: spec.threshold, + }; + case "queue_stalled": + return { + kind: "queue_stalled", + verified: false, + depth: null, + notDecreasingStreak: 0, + ticks: spec.ticks, + }; + case "queue_oldest_age": + return { + kind: "queue_oldest_age", + verified: false, + ageMs: null, + thresholdMinutes: spec.thresholdMinutes, + }; + case "error_recurrence": + return { kind: "error_recurrence", verified: false, countSince: 0 }; + case "health_recovery": + return { kind: "health_recovery", verified: false, severity: null }; + default: { + const unreachable: never = spec; + throw new Error(`Unhandled watch kind: ${JSON.stringify(unreachable)}`); + } + } +} diff --git a/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts b/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts new file mode 100644 index 00000000000..d317426e25e --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts @@ -0,0 +1,72 @@ +/** The error-recurrence condition family. */ + +import { ErrorId } from "@trigger.dev/core/v3/isomorphic"; +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; + +/** + * The model cites the API error id (`error_`) but ClickHouse stores the raw + * fingerprint. Same normalization the errors API route uses. + */ +export function normalizeErrorFingerprint(fingerprint: string): string { + return ErrorId.toId(fingerprint); +} + +/** + * Satisfied on the first occurrence proven to be after the server-set `since`, which is + * never caller-set. The facts carry the precision of what they claim. + */ +export async function checkErrorRecurrence( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const fingerprint = normalizeErrorFingerprint(spec.fingerprint); + const recurrence = await deps.readErrorRecurrence(fingerprint, input.since); + const base = { fingerprint, since: input.since.toISOString() }; + + const quiet: WatchObservedOutcome = { + kind: "error_recurrence", + verified: true, + countSince: 0, + }; + + if (!recurrence) { + return { + result: "pending", + facts: { ...base, countSince: 0, lastSeenAt: null }, + observed: quiet, + }; + } + + const lastSeenAt = recurrence.lastSeenAt?.toISOString() ?? null; + + if (!recurrence.occurredAt) { + return { + result: "pending", + facts: { ...base, countSince: 0, lastSeenAt }, + observed: quiet, + }; + } + + return { + result: "satisfied", + facts: { + ...base, + occurredAt: recurrence.occurredAt.toISOString(), + occurredAtPrecision: recurrence.occurredAtPrecision, + countSince: recurrence.countSince, + countApproximate: recurrence.countApproximate, + lastSeenAt, + }, + observed: { + kind: "error_recurrence", + verified: true, + countSince: recurrence.countSince, + }, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts b/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts new file mode 100644 index 00000000000..443597e5fa9 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts @@ -0,0 +1,49 @@ +/** The health-recovery condition family. */ + +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; + +/** + * Satisfied only when the health report is both trustworthy and `ok`. An untrustworthy + * report can never fire a recovery. + */ +export async function checkHealthRecovery( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const health = await deps.readHealth(); + if (!health) { + return { + result: "unavailable", + facts: { report: spec.report, reason: "report_unavailable" }, + observed: { kind: "health_recovery", verified: false, severity: null }, + }; + } + + const facts = { + report: spec.report, + fromSeverity: spec.fromSeverity, + severity: health.severity, + trustworthy: health.trustworthy, + }; + + if (!health.trustworthy) { + // An untrustworthy report is not an observation of the severity, so record none. + return { + result: "pending", + facts: { ...facts, reason: "untrustworthy" }, + observed: { kind: "health_recovery", verified: false, severity: null }, + }; + } + + return { + result: health.severity === "ok" ? "satisfied" : "pending", + facts, + observed: { kind: "health_recovery", verified: true, severity: health.severity }, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts b/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts new file mode 100644 index 00000000000..7c2011ab04b --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts @@ -0,0 +1,89 @@ +/** + * The reading turn of a consented watch, kicked from here because the wake turn has no token. + * The token is for `watch.userId` against `watch.environmentId`, off the row, never a body. + */ + +import type { WatchInvestigateAction } from "@internal/dashboard-agent"; +import { type Watch } from "@internal/dashboard-agent-db"; +import { watchResultNeedsAttention } from "@internal/dashboard-agent-contracts"; +import { ApiClient } from "@trigger.dev/core/v3"; +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + dashboardAgentApiOrigin, + dashboardAgentEnvironmentName, + mintDashboardAgentUserActorToken, +} from "~/services/dashboardAgent.server"; +import { env } from "~/env.server"; + +/** + * Whether the consent covers this resolved watch. The same contracts call the agent's wake + * makes, so the two can't disagree about which outcomes need attention. + */ +export function watchWantsInvestigation(watch: Watch): boolean { + if (!watch.investigateOnAttention) return false; + if (!watch.resolution) return false; + return watchResultNeedsAttention({ + kind: watch.spec.kind, + resolution: watch.resolution, + outcome: watch.observedOutcome, + }); +} + +/** The action the agent receives. Stable id, so a retried kick is a no-op. */ +export function watchInvestigateAction(watch: Watch): WatchInvestigateAction { + return { + type: "watch.investigate" as const, + id: `watch:${watch.id}:${watch.status}:investigate`, + watchId: watch.id, + identity: watch.identity, + spec: watch.spec, + facts: (watch.lastResult ?? {}) as Record, + resolution: watch.resolution ?? undefined, + observed: watch.observedOutcome ?? undefined, + note: watch.spec.note, + }; +} + +/** + * Send the kick. Throws on failure, and every caller treats it as best-effort: the wake is + * already delivered, so nothing here may retry or invalidate it. + */ +export async function kickWatchInvestigation(params: { + watch: Watch; + environment: AuthenticatedEnvironment; +}): Promise { + const { watch, environment } = params; + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + // The watch's immutable tenancy plus the delegated token that lets the turn read. + const metadata = { + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + projectRef: watch.projectRef ?? environment.project.externalRef, + environmentName: dashboardAgentEnvironmentName(environment.type), + apiOrigin, + userActorToken: await mintDashboardAgentUserActorToken(watch.userId, { + environmentId: watch.environmentId, + }), + }; + + const apiClient = new ApiClient(apiOrigin, accessToken); + await apiClient.appendToSessionStream( + // Sessions are addressable by externalId, which is the chat id. + watch.chatId, + "in", + JSON.stringify({ + kind: "message", + payload: { + chatId: watch.chatId, + trigger: "action", + action: watchInvestigateAction(watch), + metadata, + }, + }) + ); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts b/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts new file mode 100644 index 00000000000..3e251818655 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts @@ -0,0 +1,314 @@ +/** + * The queue condition family: drain, the two depth thresholds, stall and oldest age. + * They share one freshness fence — a claim of quiet needs a reading that describes now. + */ + +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import { + formatMs, + type WatchCheckDeps, + type WatchCheckInput, + type WatchCheckOutcome, + type WatchQueueDepth, +} from "./dashboardAgentWatchCheckBase"; + +/** + * The queue-depth read both threshold kinds share. A missing queue is `terminal_unsatisfied`, + * an unreadable or stale-low depth is `unavailable`, and a stale-high one is approximate. + */ +async function readDepthOrOutcome(args: { + queue: string; + deps: WatchCheckDeps; + /** The observation to record when there is no usable reading. */ + unobserved: (verified: boolean) => WatchObservedOutcome; + /** A non-current reading at or under this is refused; one above it passes through. */ + quietLine: number; + /** + * Stateful kinds only: no non-current reading is usable, because a phantom sample would + * enter the streak as if it had been observed now. + */ + requireCurrent?: boolean; +}): Promise< + | { ok: true; depth: WatchQueueDepth; facts: Record } + | { ok: false; outcome: WatchCheckOutcome } +> { + const { queue, deps, unobserved, quietLine } = args; + const depth = await deps.readQueueDepth(queue); + + if (depth === null) { + // Only a missing queue is terminal, not an unreadable depth. + const exists = await deps.queueExists(queue); + if (!exists) { + return { + ok: false, + outcome: { + result: "terminal_unsatisfied", + facts: { queue, reason: "queue_not_found" }, + observed: unobserved(true), + }, + }; + } + return { + ok: false, + outcome: { + result: "unavailable", + facts: { queue, reason: "depth_unavailable" }, + observed: unobserved(false), + }, + }; + } + + const facts = { + queue, + depth: depth.depth, + depthSource: depth.source, + depthAsOf: depth.asOf?.toISOString() ?? null, + depthApproximate: !depth.current, + }; + + // A claim of quiet needs a reading that describes now: a stale empty bucket is never + // read as drained. + if (!depth.current && (args.requireCurrent || depth.depth <= quietLine)) { + return { + ok: false, + outcome: { + result: "unavailable", + facts: { ...facts, reason: "depth_stale" }, + observed: unobserved(false), + }, + }; + } + + return { ok: true, depth, facts }; +} + +/** + * Satisfied when the queue's current pending count is 0. The observation carries the depth + * read, so a window completing without a drain needs no second read. + */ +export async function checkBacklogDrain( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ kind: "backlog_drain", verified, depth: null }), + quietLine: 0, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth === 0 ? "satisfied" : "pending", + facts: read.facts, + observed: { kind: "backlog_drain", verified: true, depth: read.depth.depth }, + }; +} + +/** + * Satisfied when the pending count rises above `threshold`. No `terminal_unsatisfied` on a + * live queue: only the queue disappearing makes the condition impossible. + */ +export async function checkQueueDepthAbove( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_depth_above", + verified, + depth: null, + threshold: spec.threshold, + }), + quietLine: spec.threshold, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth > spec.threshold ? "satisfied" : "pending", + facts: { ...read.facts, threshold: spec.threshold }, + observed: { + kind: "queue_depth_above", + verified: true, + depth: read.depth.depth, + threshold: spec.threshold, + }, + }; +} + +/** + * The mirror of `queue_depth_above`: satisfied at or under `threshold`, which is also the quiet + * line for the freshness fence. Only the queue disappearing is terminal. + */ +export async function checkQueueDepthBelow( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_depth_below", + verified, + depth: null, + threshold: spec.threshold, + }), + quietLine: spec.threshold, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth <= spec.threshold ? "satisfied" : "pending", + facts: { ...read.facts, threshold: spec.threshold }, + observed: { + kind: "queue_depth_below", + verified: true, + depth: read.depth.depth, + threshold: spec.threshold, + }, + }; +} + +/** The stall state one check hands the next, read out of the previous facts. */ +type WatchStallState = { depth: number; notDecreasingStreak: number }; + +function readStallState( + previous: Record | null | undefined +): WatchStallState | null { + if (!previous) return null; + const depth = previous.depth; + if (typeof depth !== "number" || !Number.isFinite(depth)) return null; + const streak = previous.notDecreasingStreak; + return { + depth, + notDecreasingStreak: typeof streak === "number" && Number.isFinite(streak) ? streak : 0, + }; +} + +/** + * Satisfied when the depth fails to decrease for `ticks` consecutive checks with runs queued. + * The streak lives only in `input.previous`, a gap freezes it, and depth 0 resets it. + */ +export async function checkQueueStalled( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const previous = readStallState(input.previous); + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_stalled", + verified, + depth: null, + // Carry the streak through an unusable check. + notDecreasingStreak: previous?.notDecreasingStreak ?? 0, + ticks: spec.ticks, + }), + quietLine: 0, + requireCurrent: true, + }); + if (!read.ok) return read.outcome; + + const depth = read.depth.depth; + // A first observation has nothing to compare against, so it isn't a stalled tick. + const notDecreasingStreak = + depth === 0 || previous === null + ? 0 + : depth >= previous.depth + ? previous.notDecreasingStreak + 1 + : 0; + + const facts = { + ...read.facts, + previousDepth: previous?.depth ?? null, + notDecreasingStreak, + ticks: spec.ticks, + }; + + return { + result: depth > 0 && notDecreasingStreak >= spec.ticks ? "satisfied" : "pending", + facts, + observed: { + kind: "queue_stalled", + verified: true, + depth, + notDecreasingStreak, + ticks: spec.ticks, + }, + }; +} + +/** + * Satisfied when the oldest waiting run has waited longer than the SLA. Any stale reading is + * `unavailable` rather than compared, and an empty queue is `pending`. + */ +export async function checkQueueOldestAge( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const thresholdMs = spec.thresholdMinutes * 60_000; + const unobserved = (verified: boolean): WatchObservedOutcome => ({ + kind: "queue_oldest_age", + verified, + ageMs: null, + thresholdMinutes: spec.thresholdMinutes, + }); + + const gone = (): WatchCheckOutcome => ({ + result: "terminal_unsatisfied", + facts: { queue: spec.queue, reason: "queue_not_found" }, + observed: unobserved(true), + }); + + const reading = await deps.readQueueOldestAge(spec.queue); + + if (reading === null) { + if (!(await deps.queueExists(spec.queue))) return gone(); + return { + result: "unavailable", + facts: { queue: spec.queue, reason: "age_unavailable" }, + observed: unobserved(false), + }; + } + + // Nothing waiting reads the same as a deleted queue, and only the second is terminal. + if (reading.ageMs === null && !(await deps.queueExists(spec.queue))) return gone(); + + const facts = { + queue: spec.queue, + ageMs: reading.ageMs, + ageLabel: reading.ageMs === null ? null : formatMs(reading.ageMs), + ageSource: reading.source, + ageAsOf: reading.asOf?.toISOString() ?? null, + thresholdMinutes: spec.thresholdMinutes, + }; + + if (!reading.current) { + return { + result: "unavailable", + facts: { ...facts, reason: "age_stale" }, + observed: unobserved(false), + }; + } + + const observed: WatchObservedOutcome = { + kind: "queue_oldest_age", + verified: true, + ageMs: reading.ageMs, + thresholdMinutes: spec.thresholdMinutes, + }; + + return { + result: reading.ageMs !== null && reading.ageMs > thresholdMs ? "satisfied" : "pending", + facts, + observed, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts b/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts new file mode 100644 index 00000000000..ceccbb43f0f --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts @@ -0,0 +1,240 @@ +/** + * The run condition family: start, finished, failed. All three read one run row and + * label the wait with what the data actually supports. + */ + +import { + watchRunDisposition, + type WatchObservedOutcome, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { + formatMs, + type WatchCheckDeps, + type WatchCheckInput, + type WatchCheckOutcome, + type WatchRunRow, +} from "./dashboardAgentWatchCheckBase"; + +// Mirrors ~/v3/taskStatus. Kept local so this module has no server-side import. + +const FINAL_STATUSES = new Set([ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]); + +/** + * Statuses whose `queuedAt` is a leftover from the first enqueue, since resume/retry + * re-enqueues don't restamp it, so a wait computed from it isn't this attempt's. + */ +const STALE_QUEUED_AT_STATUSES = new Set(["WAITING_TO_RESUME", "RETRYING_AFTER_FAILURE", "PAUSED"]); + +export function isTerminalRunStatus(status: string): boolean { + return FINAL_STATUSES.has(status); +} + +/** Which timestamp a wait was measured from. */ +export type WatchWaitBasis = "queued_at" | "delay_until" | "created_at"; + +/** + * The wait a run has accumulated, labelled with what the data supports. A resumed, retried or + * paused run's stale `queuedAt` is never measured from. + */ +export function describeRunWait( + run: WatchRunRow, + now: Date +): { + waitMs: number | null; + waitBasis: WatchWaitBasis; + waitLabel: string; + /** True only when the wait is this attempt's queue wait. */ + queueWaitReliable: boolean; +} { + const queueWaitReliable = run.queuedAt !== null && !STALE_QUEUED_AT_STATUSES.has(run.status); + const end = run.startedAt ?? now; + + if (run.queuedAt && queueWaitReliable) { + const waitMs = Math.max(0, end.getTime() - run.queuedAt.getTime()); + return { + waitMs, + waitBasis: "queued_at", + waitLabel: `queued for ${formatMs(waitMs)}`, + queueWaitReliable, + }; + } + + if (run.delayUntil && run.delayUntil.getTime() > now.getTime()) { + return { + waitMs: null, + waitBasis: "delay_until", + waitLabel: `scheduled to start at ${run.delayUntil.toISOString()}`, + queueWaitReliable, + }; + } + + // No `queuedAt`, or one from an earlier attempt: fall back to the run's age. + const waitMs = Math.max(0, end.getTime() - run.createdAt.getTime()); + const resumeOrRetry = run.queuedAt !== null; + return { + waitMs, + waitBasis: "created_at", + waitLabel: resumeOrRetry + ? `waiting to ${run.status === "RETRYING_AFTER_FAILURE" ? "retry" : "resume"}; time from creation: ${formatMs(waitMs)}` + : `time from creation: ${formatMs(waitMs)}`, + queueWaitReliable, + }; +} + +/** + * Satisfied the moment `startedAt` exists, whatever the current status. Terminal with no + * `startedAt` can never start, so it is `terminal_unsatisfied`. + */ +export async function checkRunStart( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + // Existence was validated at creation, so the run is gone and can never start. + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_start", verified: true, status: null, started: false }, + }; + } + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + status: run.status, + queue: run.queue, + startedAt: run.startedAt?.toISOString() ?? null, + queuedAt: run.queuedAt?.toISOString() ?? null, + ...wait, + }; + const observed: WatchObservedOutcome = { + kind: "run_start", + verified: true, + status: run.status, + started: run.startedAt !== null, + }; + + if (run.startedAt) return { result: "satisfied", facts, observed }; + if (isTerminalRunStatus(run.status)) { + return { + result: "terminal_unsatisfied", + facts: { ...facts, reason: "never_started" }, + observed, + }; + } + return { result: "pending", facts, observed }; +} + +/** + * Satisfied on any terminal status. Finished and failed are both `condition_met`, so only + * `observed.finalStatus` separates them. + */ +export async function checkRunFinished( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_finished", verified: true, finalStatus: null, durationMs: null }, + }; + } + + const finished = isTerminalRunStatus(run.status); + // Execution duration only. The queue wait is reported separately. + const durationMs = + run.startedAt && run.completedAt + ? Math.max(0, run.completedAt.getTime() - run.startedAt.getTime()) + : null; + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + outcome: run.status, + startedAt: run.startedAt?.toISOString() ?? null, + completedAt: run.completedAt?.toISOString() ?? null, + durationMs, + durationLabel: durationMs === null ? null : formatMs(durationMs), + ...wait, + }; + + return { + result: finished ? "satisfied" : "pending", + facts, + observed: { + kind: "run_finished", + verified: true, + // Only a terminal status is a final status. + finalStatus: finished ? run.status : null, + durationMs, + }, + }; +} + +/** + * A failing terminal status satisfies this; a successful completion or a cancellation makes + * the condition impossible rather than merely unmet. + */ +export async function checkRunFailed( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_failed", verified: true, finalStatus: null, durationMs: null }, + }; + } + + const finished = isTerminalRunStatus(run.status); + const durationMs = + run.startedAt && run.completedAt + ? Math.max(0, run.completedAt.getTime() - run.startedAt.getTime()) + : null; + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + outcome: run.status, + startedAt: run.startedAt?.toISOString() ?? null, + completedAt: run.completedAt?.toISOString() ?? null, + durationMs, + durationLabel: durationMs === null ? null : formatMs(durationMs), + ...wait, + }; + const observed: WatchObservedOutcome = { + kind: "run_failed", + verified: true, + finalStatus: finished ? run.status : null, + durationMs, + }; + + if (!finished) return { result: "pending", facts, observed }; + + return { + result: watchRunDisposition(run.status) === "failed" ? "satisfied" : "terminal_unsatisfied", + facts: { + ...facts, + ...(watchRunDisposition(run.status) === "failed" ? {} : { reason: "cannot_fail_now" }), + }, + observed, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts b/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts new file mode 100644 index 00000000000..45358854413 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts @@ -0,0 +1,529 @@ +/** + * The watch backstop for a dead tick chain. Finalization runs even with no agent project, or rows + * would stay `active` forever; what can't be handed over stays owed. Guarded, so a re-run no-ops. + */ + +import { + cancelWatch, + claimWatchAlertDispatch, + deleteTerminalWatchesOlderThan, + deleteWatchSubmissionsOlderThan, + listExpiredActiveWatches, + listWatchBatchGroupsToArm, + listWatchesAwaitingDelivery, + releaseWatchAlertDispatch, + transitionWatchCondition, + type Watch, + type WatchBatchGroup, +} from "@internal/dashboard-agent-db"; +import { + watchResolutionForCheck, + watchResolutionToWireStatus, + type WatchObservedOutcome, + type WatchResolution, +} from "@internal/dashboard-agent-contracts"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { enqueueWatchFiredAlert } from "~/services/dashboardAgentWatchAlerts.server"; +import { + checkWatch, + previousCheckFacts, + type WatchCheckDeps, + type WatchCheckOutcome, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { mapWithConcurrency, shareReads } from "~/services/dashboardAgentWatchBatch.server"; +import { isDashboardAgentConfigured } from "~/services/dashboardAgent.server"; +import { + armDashboardAgentWatchBatch, + authorizeWatchEnvironment, + scheduleWatchDelivery, + type WatchAuthorization, +} from "~/services/dashboardAgentWatches.server"; +import { logger } from "~/services/logger.server"; + +/** + * How long past `expiresAt` a watch is left to the tick chain. Only has to cover a late + * tick: the chain's own final check happens within a cadence of the deadline. + */ +export const WATCH_EXPIRY_GRACE_MS = 2 * 60 * 1000; + +/** + * How long a resolved watch may owe its wake before the sweep recovers it. Long enough that + * the recovery can't race a delivery still in flight. + */ +export const WATCH_DELIVERY_GRACE_MS = 5 * 60 * 1000; + +/** Per-run cap for each half of the sweep. Oldest first, so the rest land next run. */ +const SWEEP_BATCH_LIMIT = 100; + +/** How long a terminal watch is kept. Its outcome also lives in the chat transcript. */ +export const WATCH_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; + +/** Higher than the other caps: retention is one statement, not a row-at-a-time loop. */ +const RETENTION_BATCH_LIMIT = 500; + +/** + * How many rows one sweep handles at once. An incident expires a whole group together, and a + * bound is what stops one slow tenant spending the entire visibility window. + */ +const SWEEP_CONCURRENCY = 8; + +/** What one finalization did. */ +export type WatchFinalizeOutcome = + | "fired" + | "expired" + /** The user lost access: cancelled, and deliberately not narrated. */ + | "cancelled" + /** A tick (or another sweep) resolved it first. */ + | "already_resolved"; + +export type WatchSweepResult = { + /** Overdue active rows seen. */ + overdue: number; + fired: number; + expired: number; + cancelled: number; + alreadyResolved: number; + /** Resolved rows whose wake was still owed. */ + undelivered: number; + /** Wakes handed back to the watcher task. */ + redelivered: number; + /** Decided but not handed over, with no agent project. They stay owed. */ + deliveryDeferred: number; + /** Long-terminal rows dropped by retention. */ + purged: number; + /** Ledger rows dropped by retention. */ + purgedSubmissions: number; + failed: number; +}; + +export type WatchSweepDeps = { + now?: () => Date; + limit?: number; + /** Overdue `active` rows. */ + listOverdue?: (params: { now: Date; limit: number }) => Promise; + /** Resolved rows whose wake is still owed. */ + listAwaitingDelivery?: (params: { olderThan: Date; limit: number }) => Promise; + /** Re-authorization of the watch's initiating user. */ + authorize?: (watch: Watch) => Promise; + /** The environment readers the final check runs against. */ + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + /** Hand the wake back to the watcher task. Must throw if it can't be scheduled. */ + deliver?: (watch: Watch) => Promise; + /** Gates the delivery half only. Finalization never depends on it. */ + configured?: () => boolean; + /** Drop terminal rows older than `before`. Returns how many went. */ + purgeTerminal?: (params: { before: Date; limit: number }) => Promise; + /** Drop submission-ledger rows older than `before`. */ + purgeSubmissions?: (params: { before: Date; limit: number }) => Promise; + /** How many rows are handled at once. */ + concurrency?: number; +}; + +/** + * One re-authorization per (user, org, project, environment) for the whole sweep. An incident + * expires a group together, and every row of it names the same access question. + */ +function authorizeOncePerSweep( + deps: WatchSweepDeps +): (watch: Watch) => Promise { + const authorize = deps.authorize ?? defaultAuthorize; + const seen = new Map>(); + return (watch) => { + const key = `${watch.userId}:${watch.organizationId}:${watch.projectId}:${watch.environmentId}`; + const cached = seen.get(key); + if (cached) return cached; + const pending = authorize(watch); + seen.set(key, pending); + return pending; + }; +} + +/** + * One set of readers per environment, read-shared the way the batch's are: `now` is fixed for + * the sweep, so the same read can't answer two ways. + */ +function readersOncePerSweep( + deps: WatchSweepDeps +): (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps { + const build = deps.checkDeps ?? watchCheckDeps; + const seen = new Map(); + return (environment, now) => { + const cached = seen.get(environment.id); + if (cached) return cached; + const readers = shareReads(build(environment, now)); + seen.set(environment.id, readers); + return readers; + }; +} + +/** + * Facts for a swept expiry, in the same shape the tick writes. `verified: false` means the + * condition couldn't be evaluated, and carries the last observation instead. + */ +function expiredFacts( + watch: Watch, + args: { verified: boolean; reason: string; facts?: Record } +): Record { + return { + verified: args.verified, + reason: args.reason, + expiredAt: watch.expiresAt.toISOString(), + checks: watch.tickCount, + ...(args.verified + ? (args.facts ?? {}) + : { + lastObservedAt: watch.lastCheckedAt?.toISOString(), + lastObservation: watch.lastResult, + }), + }; +} + +/** + * How a final check's verdict resolves the row. The final read is a real evaluation, so only + * `pending` and `unavailable` become `window_completed`. `watchResolutionForCheck` owns that. + */ +function resolutionFor( + watch: Watch, + outcome: WatchCheckOutcome +): { + resolution: WatchResolution; + observed: WatchObservedOutcome; + facts: Record; +} { + // Always non-null here: this is the boundary evaluation. + const resolution = watchResolutionForCheck(outcome.result, true)!; + + switch (outcome.result) { + case "satisfied": + return { + resolution, + observed: outcome.observed, + facts: { verified: true, ...outcome.facts }, + }; + case "terminal_unsatisfied": + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { + verified: true, + reason: "terminal_unsatisfied", + facts: outcome.facts, + }), + }; + case "pending": + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { + verified: true, + reason: "not_met_by_expiry", + facts: outcome.facts, + }), + }; + default: + // The check couldn't run, so the window completes unverified. + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { verified: false, reason: "unverified_at_expiry" }), + }; + } +} + +/** + * Finalize one overdue watch. Re-authorization comes first, before the final check reads + * anything; `canDeliver: false` stops at the resolution, leaving the wake owed. + */ +export async function finalizeOverdueWatch( + watch: Watch, + deps: WatchSweepDeps & { canDeliver?: boolean } = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const authorize = deps.authorize ?? defaultAuthorize; + const buildCheckDeps = deps.checkDeps ?? watchCheckDeps; + const deliver = deps.deliver ?? scheduleWatchDelivery; + const canDeliver = deps.canDeliver ?? true; + + const authorization = await authorize(watch); + if (!authorization.ok) { + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "access_revoked" }); + logger.info("Dashboard agent watch sweep: cancelled a watch whose access was revoked", { + watchId: watch.id, + }); + return "cancelled"; + } + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const outcome = await checkWatch( + watch.spec, + buildCheckDeps(authorization.environment, now), + // A stateful condition is a transition across checks, so the boundary evaluation needs + // the previous facts to see one — and to record what it saw, not a reset. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch sweep: the final check failed", { + watchId: watch.id, + error, + }) + ); + + const resolved = resolutionFor(watch, outcome); + const transitioned = await transitionWatchCondition(dashboardAgentDb, { + id: watch.id, + resolution: resolved.resolution, + observedOutcome: resolved.observed, + lastResult: resolved.facts, + }); + + // Guarded on `active`: a tick that resolved it first keeps its outcome and delivery. + if (!transitioned) return "already_resolved"; + + if (resolved.resolution === "condition_met") { + // Claimed first, exactly as the fire callback does: the wake this sweep is about to + // schedule reports the same fired watch, and an unclaimed row alerts a second time. + const claimed = await claimWatchAlertDispatch(dashboardAgentDb, { + id: watch.id, + terminalStatus: "fired", + }); + if (claimed) { + try { + await enqueueWatchFiredAlert(transitioned, "fired"); + } catch (error) { + await releaseWatchAlertDispatch(dashboardAgentDb, { + id: watch.id, + terminalStatus: "fired", + }); + logger.error("Dashboard agent watch sweep: failed to enqueue the fired alert", { + watchId: watch.id, + error, + }); + } + } + } + + // Throws if it can't be scheduled, leaving the row terminal with its delivery owed. + if (canDeliver) await deliver(transitioned); + return watchResolutionToWireStatus(resolved.resolution); +} + +/** + * Recover one owed wake, unconditionally: this sweep can't tell whether the user was already + * told. Whether the wake needs prose is decided where the transcript can be read. + */ +export async function recoverWatchDelivery(watch: Watch, deps: WatchSweepDeps = {}): Promise { + const deliver = deps.deliver ?? scheduleWatchDelivery; + await deliver(watch); +} + +/** + * One sweep: finalize what is overdue, then recover what was never delivered. Each row is + * handled on its own, and the run throws at the end if any failed so the job is retried. + */ +export async function sweepDashboardAgentWatches( + deps: WatchSweepDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const limit = deps.limit ?? SWEEP_BATCH_LIMIT; + const configured = deps.configured ?? isDashboardAgentConfigured; + const listOverdue = + deps.listOverdue ?? ((params) => listExpiredActiveWatches(dashboardAgentDb, params)); + const listAwaitingDelivery = + deps.listAwaitingDelivery ?? + ((params) => listWatchesAwaitingDelivery(dashboardAgentDb, params)); + const purgeTerminal = + deps.purgeTerminal ?? ((params) => deleteTerminalWatchesOlderThan(dashboardAgentDb, params)); + const purgeSubmissions = + deps.purgeSubmissions ?? + ((params) => deleteWatchSubmissionsOlderThan(dashboardAgentDb, params)); + + const result: WatchSweepResult = { + overdue: 0, + fired: 0, + expired: 0, + cancelled: 0, + alreadyResolved: 0, + undelivered: 0, + redelivered: 0, + deliveryDeferred: 0, + purged: 0, + purgedSubmissions: 0, + failed: 0, + }; + + // Gates the hand-off only: the rows still have to be finalized. + const canDeliver = configured(); + if (!canDeliver) { + logger.warn( + "Dashboard agent watch sweep: the agent isn't configured, so wakes can't be delivered — finalizing only" + ); + } + + const overdue = await listOverdue({ + now: new Date(now.getTime() - WATCH_EXPIRY_GRACE_MS), + limit, + }); + result.overdue = overdue.length; + + // The authorization and the readers are resolved once for the whole sweep, so a group of + // expiries costs one of each rather than one per row. + const perSweep: WatchSweepDeps & { canDeliver: boolean } = { + ...deps, + now: () => now, + canDeliver, + authorize: authorizeOncePerSweep(deps), + checkDeps: readersOncePerSweep(deps), + }; + + const finalized = await mapWithConcurrency( + overdue, + deps.concurrency ?? SWEEP_CONCURRENCY, + async (watch) => { + try { + return await finalizeOverdueWatch(watch, perSweep); + } catch (error) { + logger.error("Dashboard agent watch sweep: failed to finalize a watch", { + watchId: watch.id, + error, + }); + return null; + } + } + ); + + for (const outcome of finalized) { + if (outcome === null) result.failed++; + else if (outcome === "fired") result.fired++; + else if (outcome === "expired") result.expired++; + else if (outcome === "cancelled") result.cancelled++; + else result.alreadyResolved++; + // Resolved, but nothing carried the wake away: it stays owed. + if (!canDeliver && (outcome === "fired" || outcome === "expired")) { + result.deliveryDeferred++; + } + } + + // Skipped without an agent project: the rows keep their owed wake for the next sweep. + if (canDeliver) { + const owed = await listAwaitingDelivery({ + olderThan: new Date(now.getTime() - WATCH_DELIVERY_GRACE_MS), + limit, + }); + result.undelivered = owed.length; + + const recovered = await mapWithConcurrency( + owed, + deps.concurrency ?? SWEEP_CONCURRENCY, + async (watch) => { + try { + await recoverWatchDelivery(watch, deps); + return true; + } catch (error) { + logger.error("Dashboard agent watch sweep: failed to recover a wake", { + watchId: watch.id, + error, + }); + return false; + } + } + ); + + for (const ok of recovered) { + if (ok) result.redelivered++; + else result.failed++; + } + } + + // Retention runs last, over rows both halves are finished with. Its own try/catch so a + // lost retention pass can't mask the other failures. + try { + const before = new Date(now.getTime() - WATCH_RETENTION_MS); + result.purged = await purgeTerminal({ before, limit: RETENTION_BATCH_LIMIT }); + // The ledger's rows age out on the same window: past it no client is still retrying. + result.purgedSubmissions = await purgeSubmissions({ before, limit: RETENTION_BATCH_LIMIT }); + } catch (error) { + result.failed++; + logger.error("Dashboard agent watch sweep: failed to purge terminal watches", { error }); + } + + if (result.failed > 0) { + throw new Error(`The dashboard agent watch sweep failed on ${result.failed} watches`); + } + + return result; +} + +export type WatchBatchRearmResult = { + /** Groups with active watches and no live chain. */ + stale: number; + armed: number; + failed: number; +}; + +export type WatchBatchRearmDeps = { + now?: () => Date; + limit?: number; + /** Groups whose chain is missing, stopped, or has gone silent. */ + listGroups?: (params: { now: Date; limit: number }) => Promise; + /** Start a chain for one group. */ + arm?: (params: { + environmentId: string; + cadenceMinutes: number; + now?: Date; + }) => Promise<{ running: boolean }>; + configured?: () => boolean; +}; + +const REARM_BATCH_LIMIT = 200; + +/** + * Re-arm batch chains that died. `armWatchBatch` re-checks the same timestamp in the statement + * that arms, so racing a merely slow chain arms nothing and two runs start one chain. + */ +export async function rearmDashboardAgentWatchBatches( + deps: WatchBatchRearmDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const limit = deps.limit ?? REARM_BATCH_LIMIT; + const configured = deps.configured ?? isDashboardAgentConfigured; + const listGroups = + deps.listGroups ?? ((params) => listWatchBatchGroupsToArm(dashboardAgentDb, params)); + const arm = deps.arm ?? armDashboardAgentWatchBatch; + + const result: WatchBatchRearmResult = { stale: 0, armed: 0, failed: 0 }; + + // Nothing to trigger a chain into. The expiry half still finalizes the rows. + if (!configured()) return result; + + const groups = await listGroups({ now, limit }); + result.stale = groups.length; + + for (const group of groups) { + try { + const { running } = await arm({ ...group, now }); + if (running) result.armed++; + } catch (error) { + result.failed++; + logger.error("Dashboard agent watch sweep: failed to re-arm a batch chain", { + ...group, + error, + }); + } + } + + if (result.failed > 0) { + throw new Error(`The dashboard agent batch re-arm failed on ${result.failed} groups`); + } + + return result; +} + +function defaultAuthorize(watch: Watch): Promise { + return authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchToken.server.ts b/apps/webapp/app/services/dashboardAgentWatchToken.server.ts new file mode 100644 index 00000000000..dc56691f560 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchToken.server.ts @@ -0,0 +1,181 @@ +import { generateJWT, validateJWT } from "@trigger.dev/core/v3/jwt"; +import { env } from "~/env.server"; + +/** + * The credential the watcher task presents to the private check endpoint. It names one watch, is + * kept apart from the UAT by a disjoint prefix and `kind`, and is re-minted, never persisted. + */ + +export const WATCH_TOKEN_PREFIX = "tr_daw_"; + +/** Distinguishes a watch token from every other SESSION_SECRET-signed JWT. */ +const WATCH_TOKEN_KIND = "dashboard_agent_watch"; + +/** Mirrors the UAT's `act.client` with a value no UAT uses. Verified, so no replay. */ +const WATCH_TOKEN_CLIENT = "dashboard-agent-watch"; + +/** + * How long past `expiresAt` the token stays valid. The final check happens after the + * deadline, so the token has to outlive the watch by enough to cover a late tick. + */ +export const WATCH_TOKEN_GRACE_MS = 60 * 60 * 1000; + +export type WatchTokenClaims = { + watchId: string; + /** Token expiry (seconds since epoch), i.e. `expiresAt` + grace. */ + expiresAtSeconds: number; +}; + +export function isDashboardAgentWatchToken(token: string): boolean { + return token.startsWith(WATCH_TOKEN_PREFIX); +} + +/** Deterministic: the same inputs produce the same string. */ +export async function signDashboardAgentWatchToken( + secret: string, + opts: { watchId: string; expiresAt: Date; graceMs?: number } +): Promise { + const expirationTime = Math.floor( + (opts.expiresAt.getTime() + (opts.graceMs ?? WATCH_TOKEN_GRACE_MS)) / 1000 + ); + + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: WATCH_TOKEN_KIND, + // `sub` is the watch: a watch token never authenticates a user. + sub: opts.watchId, + act: { client: WATCH_TOKEN_CLIENT }, + }, + expirationTime, + omitIssuedAt: true, + }); + + return `${WATCH_TOKEN_PREFIX}${jwt}`; +} + +/** `undefined` for anything but a valid watch token, including a valid user-actor token. */ +export async function verifyDashboardAgentWatchToken( + secret: string, + token: string +): Promise { + if (!isDashboardAgentWatchToken(token)) return; + + const result = await validateJWT(token.slice(WATCH_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== WATCH_TOKEN_KIND) return; + if (typeof payload.sub !== "string" || payload.sub.length === 0) return; + + const act = payload.act as { client?: string } | undefined; + if (act?.client !== WATCH_TOKEN_CLIENT) return; + if (typeof payload.exp !== "number") return; + + return { watchId: payload.sub, expiresAtSeconds: payload.exp }; +} + +export function mintDashboardAgentWatchToken(opts: { + watchId: string; + expiresAt: Date; +}): Promise { + return signDashboardAgentWatchToken(env.SESSION_SECRET, opts); +} + +export function verifyWatchTokenFromRequest(token: string): Promise { + return verifyDashboardAgentWatchToken(env.SESSION_SECRET, token); +} + +/** + * Chain tokens name a whole (environment, cadence) group. The prefix is disjoint from + * `tr_daw_` rather than nested under it, so neither verifier sees the other's tokens. + */ +export const WATCH_BATCH_TOKEN_PREFIX = "tr_dab_"; + +const WATCH_BATCH_TOKEN_KIND = "dashboard_agent_watch_batch"; +const WATCH_BATCH_TOKEN_CLIENT = "dashboard-agent-watch-batch"; + +/** + * How long a chain's token lives. A chain has no deadline to pin it to, but it must still + * expire; an expired one is self-healing via the re-arm backstop. + */ +export const WATCH_BATCH_TOKEN_TTL_MS = 30 * 24 * 60 * 60 * 1000; + +export type WatchBatchTokenClaims = { environmentId: string; cadenceMinutes: number }; + +/** + * Like a watch token, a chain token names one group and carries no authority of its own: the + * batch check re-authorizes every watch's initiating user against that watch's snapshot. + */ +export async function signDashboardAgentWatchBatchToken( + secret: string, + opts: { environmentId: string; cadenceMinutes: number; expiresAt: Date } +): Promise { + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: WATCH_BATCH_TOKEN_KIND, + // The group, not a user and not a watch. + sub: `${opts.environmentId}:${opts.cadenceMinutes}`, + act: { client: WATCH_BATCH_TOKEN_CLIENT }, + }, + expirationTime: Math.floor(opts.expiresAt.getTime() / 1000), + omitIssuedAt: true, + }); + + return `${WATCH_BATCH_TOKEN_PREFIX}${jwt}`; +} + +/** `undefined` for anything that isn't a valid, unexpired chain token. */ +export async function verifyDashboardAgentWatchBatchToken( + secret: string, + token: string +): Promise { + if (!token.startsWith(WATCH_BATCH_TOKEN_PREFIX)) return; + + const result = await validateJWT(token.slice(WATCH_BATCH_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== WATCH_BATCH_TOKEN_KIND) return; + const act = payload.act as { client?: string } | undefined; + if (act?.client !== WATCH_BATCH_TOKEN_CLIENT) return; + if (typeof payload.sub !== "string") return; + + // The cadence is the last segment, split from the right so a colon in an environment id + // could never confuse it. + const separator = payload.sub.lastIndexOf(":"); + if (separator <= 0) return; + const environmentId = payload.sub.slice(0, separator); + const cadenceMinutes = Number(payload.sub.slice(separator + 1)); + if (!Number.isInteger(cadenceMinutes) || cadenceMinutes <= 0) return; + + return { environmentId, cadenceMinutes }; +} + +export function mintDashboardAgentWatchBatchToken(opts: { + environmentId: string; + cadenceMinutes: number; + now?: Date; +}): Promise { + const now = opts.now ?? new Date(); + return signDashboardAgentWatchBatchToken(env.SESSION_SECRET, { + environmentId: opts.environmentId, + cadenceMinutes: opts.cadenceMinutes, + expiresAt: new Date(now.getTime() + WATCH_BATCH_TOKEN_TTL_MS), + }); +} + +export function verifyWatchBatchTokenFromRequest( + token: string +): Promise { + return verifyDashboardAgentWatchBatchToken(env.SESSION_SECRET, token); +} + +/** The bearer value from an `Authorization: Bearer …` header, if present. */ +export function bearerToken(request: Request): string | undefined { + const raw = request.headers.get("Authorization"); + if (!raw) return undefined; + const value = raw.replace(/^Bearer /, "").trim(); + return value.length > 0 ? value : undefined; +} diff --git a/apps/webapp/app/services/dashboardAgentWatches.server.ts b/apps/webapp/app/services/dashboardAgentWatches.server.ts new file mode 100644 index 00000000000..57f38ae8bd4 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatches.server.ts @@ -0,0 +1,1175 @@ +/** + * Watches, webapp half: creation, the re-authorization a background check passes, and the + * chat-delete cascade. The row's snapshot is immutable, and no target comes from client input. + */ + +import { + MAX_ACTIVE_WATCHES_PER_CHAT, + appendChatMessageOnce, + armWatchBatch, + cancelWatch, + chatExists, + claimWatchSubmission, + createChat, + createWatch, + generateWatchId, + getChatWatchContext, + getWatch, + getWatchSubmission, + listActiveWatchesForChats as listActiveWatchesForChatsQuery, + precheckWatchCreation, + recordWatchSubmissionOutcome, + reopenWatchSubmission, + softDeleteChat, + stopWatchBatch, + type ChatWatchContext, + type PersistedWatchSpec, + type Watch, + type WatchStatus, + type WatchSubmission, +} from "@internal/dashboard-agent-db"; +import { + VIEW_BLOCK_VERSION, + WATCH_CONFIRMATION_MESSAGE_ID_PREFIX, + WATCH_REQUEST_MESSAGE_ID_PREFIX, + watchConfirmationBlockBody, + watchDraftSchema, + watchIdentity, + watchOneShotBlockBody, + watchRequestSentence, + watchSubjectLabel, + type WatchDraft, + type WatchExternalNotification, + type WatchObservedOutcome, + type WatchResolution, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { createHash } from "node:crypto"; +import { TriggerClient } from "@trigger.dev/sdk"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { authIncludeWithParent, toAuthenticated } from "~/models/runtimeEnvironment.server"; +import { isReportKey } from "~/presenters/v3/reports/report-registry"; +import { + dashboardAgentApiOrigin, + isDashboardAgentConfigured as isDashboardAgentConfiguredDefault, +} from "~/services/dashboardAgent.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { + checkWatch, + type WatchCheckDeps, + type WatchCheckOutcome, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCreationCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { normalizeErrorFingerprint } from "~/services/dashboardAgentWatchErrorChecks"; +import { subscribeUserToWatchAlerts } from "~/services/dashboardAgentWatchAlerts.server"; +import { + mintDashboardAgentWatchBatchToken, + mintDashboardAgentWatchToken, +} from "~/services/dashboardAgentWatchToken.server"; +import { canAccessDashboardAgent } from "~/v3/canAccessDashboardAgent.server"; + +/** The task that polls a watch. Lives in the agent project, triggered by us. */ +export const WATCH_TASK_ID = "dashboard-agent-watch"; + +export { MAX_ACTIVE_WATCHES_PER_CHAT }; + +export type WatchAuthorization = + | { ok: true; environment: AuthenticatedEnvironment } + | { ok: false; reason: "access_revoked" }; + +/** + * Re-authorize a watch's initiating user against the row's immutable project/environment; a + * partial pass is `access_revoked`. The membership-scoped query is the tenant floor here. + */ +export async function authorizeWatchEnvironment(params: { + userId: string; + organizationId: string; + projectId: string; + environmentId: string; +}): Promise { + // The primary, not the replica: replica lag would extend access the user has lost. + const environment = await prisma.runtimeEnvironment.findFirst({ + where: { + id: params.environmentId, + // The watch's snapshot has to still describe this environment. + projectId: params.projectId, + organizationId: params.organizationId, + archivedAt: null, + project: { deletedAt: null }, + organization: { deletedAt: null, members: { some: { userId: params.userId } } }, + OR: [ + { type: { in: ["PREVIEW", "STAGING", "PRODUCTION"] } }, + // Dev environments are per-member: only their owner may read them. + { type: "DEVELOPMENT", orgMember: { userId: params.userId } }, + ], + }, + include: authIncludeWithParent, + }); + + if (!environment) return { ok: false, reason: "access_revoked" }; + + // The gate only reads `isAdmin` while the admin preview is on, so this read is skipped + // otherwise: it runs on every watch check, batch authorization and sweep finalisation. + let isAdmin = false; + if (env.DASHBOARD_AGENT_ADMIN_PREVIEW === "1") { + // Primary for the same reason as the membership read above. + const user = await prisma.user.findFirst({ + where: { id: params.userId }, + select: { admin: true }, + }); + if (!user) return { ok: false, reason: "access_revoked" }; + isAdmin = user.admin; + } + + const allowed = await canAccessDashboardAgent({ + userId: params.userId, + isAdmin, + // A background check is never an impersonated session. + isImpersonating: false, + organizationSlug: environment.organization.slug, + orgFeatureFlags: environment.organization.featureFlags as Record | null, + }); + if (!allowed) return { ok: false, reason: "access_revoked" }; + + return { ok: true, environment: toAuthenticated(environment) }; +} + +/** + * The same authorization by environment id alone, for the creation path with no watch row + * yet. The id lookup is unscoped and proves nothing; `authorizeWatchEnvironment` is the gate. + */ +export async function authorizeWatchEnvironmentById(params: { + userId: string; + environmentId: string; +}): Promise { + const environment = await $replica.runtimeEnvironment.findFirst({ + where: { id: params.environmentId }, + select: { organizationId: true, projectId: true }, + }); + if (!environment) return null; + + const authorization = await authorizeWatchEnvironment({ + userId: params.userId, + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: params.environmentId, + }); + return authorization.ok ? authorization.environment : null; +} + +export type CreateWatchErrorCode = + | "limit_reached" + | "duplicate" + | "invalid_target" + | "chat_not_found" + | "not_configured" + | "internal"; + +/** + * Either a watch is now running (`watching: true`), or the immediate check answered and no + * row exists at all (`watching: false`), which never enters the delivery state machine. + */ +export type CreateDashboardAgentWatchResult = + | { + ok: true; + watching: true; + watchId: string; + identity: string; + status: WatchStatus; + expiresAt: Date; + /** Set when the creation-time check couldn't run. The watch is active anyway. */ + unavailable?: boolean; + } + | { + ok: true; + watching: false; + identity: string; + /** `satisfied` (already true) or `terminal_unsatisfied` (can't happen now). */ + immediate: WatchCheckOutcome; + } + | { + ok: false; + error: string; + code: CreateWatchErrorCode; + /** The watch already covering this condition, on `duplicate`. */ + existingId?: string | null; + }; + +/** The one spelling of a spec's target that the identity, the checks and the link all share. */ +function normalizeWatchSpec(spec: WatchSpec): WatchSpec { + if (spec.kind !== "error_recurrence") return spec; + return { ...spec, fingerprint: normalizeErrorFingerprint(spec.fingerprint) }; +} + +/** + * Existence check for the thing a spec points at, in this environment. `error_recurrence` + * has nothing to validate: zero occurrences so far is the normal case. + */ +async function validateWatchTarget(spec: WatchSpec, deps: WatchCheckDeps): Promise { + switch (spec.kind) { + case "run_start": + case "run_finished": + case "run_failed": + return (await deps.readRun(spec.runId)) !== null; + case "backlog_drain": + case "queue_depth_above": + case "queue_depth_below": + case "queue_stalled": + case "queue_oldest_age": + return await deps.queueExists(spec.queue); + case "error_recurrence": + return spec.fingerprint.length > 0; + case "health_recovery": + return isReportKey(spec.report); + } +} + +/** + * Create a watch for an already-authorized context. The order is load-bearing (cap, dedup, + * immediate check, create), and a first tick that can't be scheduled cancels the row. + */ +export async function createDashboardAgentWatch(params: { + environment: AuthenticatedEnvironment; + userId: string; + chatId: string; + spec: WatchSpec; + /** Consent to investigate after an attention outcome. Never inferred. */ + investigateOnAttention?: boolean; + /** Reserved by the submission ledger, so a converging retry finds the row by id. */ + watchId?: string; + now?: Date; + /** IO seams: tests inject fakes here instead of mocking the readers. */ + deps?: { + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + scheduleTick?: typeof scheduleWatchTick; + /** Skip the real trigger-config gate when a tick scheduler is injected. */ + configured?: () => boolean; + }; +}): Promise { + const { environment, userId, chatId } = params; + // Normalized before anything reads it: the page cites `error_` and the tools + // cite the bare one, and only one of the two spellings may reach the identity or the link. + const spec = normalizeWatchSpec(params.spec); + const now = params.now ?? new Date(); + // Creation reads the target on the primary; the polling checks stay on the replica. + const buildCheckDeps = params.deps?.checkDeps ?? watchCreationCheckDeps; + const scheduleTick = params.deps?.scheduleTick ?? scheduleWatchTick; + const isDashboardAgentConfigured = params.deps?.configured ?? isDashboardAgentConfiguredDefault; + const checkDeps = buildCheckDeps(environment, now); + + if (!isDashboardAgentConfigured()) { + return { + ok: false, + code: "not_configured", + error: "The dashboard agent is not configured, so watches can't be scheduled.", + }; + } + + if (!(await validateWatchTarget(spec, checkDeps))) { + return { + ok: false, + code: "invalid_target", + error: "That target doesn't exist in this environment.", + }; + } + + const identity = watchIdentity(spec); + + // Advisory only: `createWatch` below re-applies both guardrails atomically and + // stays the authority. + const precheck = await precheckWatchCreation(dashboardAgentDb, { + chatId, + projectId: environment.projectId, + environmentId: environment.id, + identity, + }); + if (!precheck.ok) return creationGuardrailError(precheck); + + // `since` is server-set so the model can't backdate a recurrence window. + const persistedSpec: PersistedWatchSpec = + spec.kind === "error_recurrence" ? { ...spec, since: now.toISOString() } : spec; + + // Answer in the same turn when the condition has already happened. + const immediate = await checkWatch(persistedSpec, checkDeps, { now, since: now }, (error) => + logger.error("Dashboard agent watch: immediate check failed", { chatId, identity, error }) + ); + + if (immediate.result === "satisfied" || immediate.result === "terminal_unsatisfied") { + // Nothing is persisted: no row means no delivery claim and no wake. + return { ok: true, watching: false, identity, immediate }; + } + + const expiresAt = new Date(now.getTime() + spec.maxHours * 60 * 60 * 1000); + + const created = await createWatch(dashboardAgentDb, { + ...(params.watchId ? { id: params.watchId } : {}), + chatId, + identity, + spec: persistedSpec, + organizationId: environment.organizationId, + projectId: environment.projectId, + // The external ref travels with the row: the agent can't translate the internal id. + projectRef: environment.project.externalRef, + environmentId: environment.id, + userId, + expiresAt, + investigateOnAttention: params.investigateOnAttention === true, + }); + + if (!created.ok) { + if (created.error === "chat_not_found") { + // The chat was deleted mid-create. The query layer re-reads it under the + // per-chat lock, so nothing was written. + return { + ok: false, + code: "chat_not_found", + error: "That chat no longer exists, so nothing is being watched.", + }; + } + return creationGuardrailError(created); + } + + const watch = created.watch; + const token = await mintDashboardAgentWatchToken({ watchId: watch.id, expiresAt }); + + try { + await scheduleTick({ + watchId: watch.id, + token, + delayMinutes: spec.checkEveryMinutes, + // Each invocation claims its own generation atomically, so the first is + // `tickCount + 1`. + tick: watch.tickCount + 1, + }); + } catch (error) { + logger.error("Dashboard agent watch: failed to schedule the first tick", { + id: watch.id, + error, + }); + // Cancelled rather than resolved, because the condition was never evaluated. + // Cancellation is silent, so no wake is sent. + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "scheduling_failed" }); + return { + ok: false, + code: "internal", + error: "The watch couldn't be scheduled. Nothing is being watched.", + }; + } + + return { + ok: true, + watching: true, + watchId: watch.id, + identity, + status: "active", + expiresAt, + ...(immediate.result === "unavailable" ? { unavailable: true } : {}), + }; +} + +/* ------------------------------------------------------------------ * + * The card submit: a durable record of the request, then the watch + * ------------------------------------------------------------------ */ + +/** A stored transcript record. Deterministic, so a retry rewrites the same bytes. */ +export type WatchTranscriptMessage = { + id: string; + role: "user" | "assistant"; + parts: unknown[]; +}; + +/** A submit can also refuse a request id that arrives carrying a different draft. */ +export type SubmitWatchErrorCode = CreateWatchErrorCode | "request_conflict"; + +export type SubmitWatchCardResult = + | { + ok: true; + chatId: string; + watching: boolean; + watchId: string | null; + /** The request record and the confirmation, in transcript order. */ + messages: WatchTranscriptMessage[]; + /** Nothing was created: this call replayed a recorded outcome. */ + repaired: boolean; + } + | { + ok: false; + code: SubmitWatchErrorCode; + error: string; + existingId?: string | null; + /** Set once a chat exists, so the caller can still open it. */ + chatId?: string; + }; + +/** + * A fresh panel's chat id, derived from the request id so a retried submit lands in the + * chat the first attempt created instead of leaving an empty one behind. + * + * The whole tenancy of the request is mixed in, not just the user: `clientRequestId` is + * client-chosen, and one user can be in several organizations, so a user id alone lets two + * organizations derive the same id — where `createChat(...).onConflictDoNothing()` keeps + * the first org's chat and the second org's records land in it. + */ +function chatIdForRequest(params: { + organizationId: string; + userId: string; + environmentId: string; + clientRequestId: string; +}): string { + const digest = createHash("sha256") + .update( + `${params.organizationId}:${params.userId}:${params.environmentId}:${params.clientRequestId}` + ) + .digest("hex"); + return `chat_${digest.slice(0, 24)}`; +} + +/** + * A spec's comparable form. `since` is server-set on every attempt, so it is excluded: + * two attempts at the same request differ by it and are still the same request. + */ +function comparableSpec(spec: WatchSpec | PersistedWatchSpec): string { + const entries = Object.entries(spec as Record) + .filter(([key]) => key !== "since") + .sort(([a], [b]) => a.localeCompare(b)); + return JSON.stringify(entries); +} + +/** + * Whether an existing watch is the one this submission asked for. A retry is byte-identical, + * so anything else — a different window, cadence, note or investigate consent — is a genuinely + * different request and still conflicts. `notifyExternally` is not on the watch row, so it is + * compared through the ledger's draft digest instead, which covers the whole configuration. + */ +function isSameWatchRequest(existing: Watch, draft: WatchDraft): boolean { + return ( + comparableSpec(existing.spec) === comparableSpec(draft.spec) && + existing.investigateOnAttention === draft.followUp.investigateOnAttention + ); +} + +/** The record of what the user confirmed. Written with no model call. */ +function requestMessage(clientRequestId: string, draft: WatchDraft): WatchTranscriptMessage { + return { + id: `${WATCH_REQUEST_MESSAGE_ID_PREFIX}${clientRequestId}`, + role: "user", + parts: [ + { type: "text", text: watchRequestSentence({ spec: draft.spec, followUp: draft.followUp }) }, + ], + }; +} + +/** The confirmation block, keyed on the watch so a repair rebuilds exactly the same record. */ +function confirmationMessage(args: { + id: string; + blockId: string; + body: Record; +}): WatchTranscriptMessage { + return { + id: args.id, + role: "assistant", + parts: [ + { + type: "data-view", + data: { + blocks: [ + { ...args.body, revision: 0, version: VIEW_BLOCK_VERSION, id: `watch:${args.blockId}` }, + ], + }, + }, + ], + }; +} + +/** + * A submitted draft's comparable digest: the whole confirmed configuration, `notifyExternally` + * included. It is user consent, and the transcript records it, so a retry that flips it is a + * different request — not something to converge on behind the durable record. + */ +function draftDigest(draft: WatchDraft): string { + return createHash("sha256") + .update( + JSON.stringify([ + comparableSpec(draft.spec), + draft.followUp.investigateOnAttention, + draft.followUp.notifyExternally, + ]) + ) + .digest("hex"); +} + +/** The recorded outcome of the external consent, replayed rather than re-decided. */ +function recordedExternalNotification(recorded: WatchSubmission): WatchExternalNotification { + if (recorded.externalNotificationStatus === "enabled") return { status: "enabled" }; + if (recorded.externalNotificationStatus === "unavailable") { + return { status: "unavailable", reason: recorded.externalNotificationReason ?? "unknown" }; + } + return { status: "not_requested" }; +} + +/** The draft the ledger recorded, which is what a replay must be built from. */ +function recordedDraft(recorded: WatchSubmission, fallback: WatchDraft): WatchDraft { + const parsed = watchDraftSchema.safeParse(recorded.draft); + return parsed.success ? parsed.data : fallback; +} + +/** The refusal record, keyed off the request so a retry's success can still follow it. */ +function refusalMessage(clientRequestId: string, error: string): WatchTranscriptMessage { + return { + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}refused:${clientRequestId}`, + role: "assistant", + parts: [{ type: "text", text: error }], + }; +} + +/** + * Submit a configured watch card, for an already-authorized environment and a chat the caller + * owns. + * + * The submission ledger is the idempotency boundary, not the transcript ids: a row keyed + * `(chatId, clientRequestId)` is written *before* the condition is evaluated, and it carries + * the outcome once there is one. So a retry looks the submission up first and replays what + * was recorded — it never re-evaluates and never creates a second operation, even after the + * first watch has fired, expired or answered in one shot. Only a `pending` row, left by an + * attempt that died before writing its outcome, is allowed to proceed, and it converges on + * the watch id reserved up front rather than creating another. + * + * The transcript ordering is the second invariant: the record of what the user confirmed is + * written before anything starts running, and the confirmation after, so a crash can leave a + * watch that is visible but unconfirmed, never one that is live and invisible. + */ +export async function submitDashboardAgentWatch(params: { + environment: AuthenticatedEnvironment; + userId: string; + organizationId: string; + /** The chat the card was submitted from. A fresh panel has none, so one is created. */ + chatId?: string; + /** Stable per card submission: both transcript records are keyed off it. */ + clientRequestId: string; + draft: WatchDraft; + now?: Date; + deps?: { + create?: typeof createDashboardAgentWatch; + subscribe?: typeof subscribeUserToWatchAlerts; + } & NonNullable[0]["deps"]>; +}): Promise { + const { environment, userId, organizationId, clientRequestId, draft } = params; + const create = params.deps?.create ?? createDashboardAgentWatch; + const subscribe = params.deps?.subscribe ?? subscribeUserToWatchAlerts; + + const chatId = + params.chatId ?? + chatIdForRequest({ + organizationId, + userId, + environmentId: environment.id, + clientRequestId, + }); + if (!params.chatId) { + // Idempotent on the id, so a retry reuses the same chat rather than making another. + await createChat(dashboardAgentDb, { + id: chatId, + organizationId, + userId, + title: `Watch ${watchSubjectLabel(draft.spec)}`, + }); + } + + const digest = draftDigest(draft); + const request = requestMessage(clientRequestId, draft); + + /** Append-once, then return. Both records are deterministic, so a replay rewrites bytes. */ + const settle = async (args: { + confirmation: WatchTranscriptMessage; + watchId: string | null; + repaired: boolean; + }): Promise => { + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: args.confirmation, + }); + return { + ok: true, + chatId, + watching: args.watchId !== null, + watchId: args.watchId, + messages: [request, args.confirmation], + repaired: args.repaired, + }; + }; + + /** `confirmed` is the draft the confirmation speaks for: the recorded one on a replay. */ + const watchingConfirmation = (args: { + watchId: string; + unavailable: boolean; + external: WatchExternalNotification; + confirmed?: WatchDraft; + }) => { + const confirmed = args.confirmed ?? draft; + return confirmationMessage({ + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}${args.watchId}`, + blockId: args.watchId, + body: watchConfirmationBlockBody({ + spec: confirmed.spec, + watchId: args.watchId, + unavailable: args.unavailable, + followUp: { + investigateOnAttention: confirmed.followUp.investigateOnAttention, + external: args.external, + }, + }), + }); + }; + + const oneShotConfirmation = ( + result: "satisfied" | "terminal_unsatisfied", + confirmed: WatchDraft = draft + ) => + confirmationMessage({ + // No watch exists, so the request id is the only stable key for a one-shot. + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}one-shot:${clientRequestId}`, + blockId: watchIdentity(confirmed.spec), + body: watchOneShotBlockBody({ spec: confirmed.spec, result }), + }); + + /** + * Rebuild the transcript from a recorded outcome. Nothing is evaluated or created, and + * the record is built from the recorded draft, never the body of this attempt: the + * durable user message states what the first attempt confirmed. + */ + const replay = async (recorded: WatchSubmission): Promise => { + const confirmed = recordedDraft(recorded, draft); + + if (recorded.state === "created" && recorded.watchId) { + return settle({ + confirmation: watchingConfirmation({ + watchId: recorded.watchId, + unavailable: recorded.unavailable, + // Recorded, never re-decided: the confirmation already in the transcript is + // append-once, so a second decision here would contradict it forever. + external: recordedExternalNotification(recorded), + confirmed, + }), + watchId: recorded.watchId, + repaired: true, + }); + } + + if (recorded.state === "immediate") { + return settle({ + confirmation: oneShotConfirmation( + recorded.immediateResult === "satisfied" ? "satisfied" : "terminal_unsatisfied", + confirmed + ), + watchId: null, + repaired: true, + }); + } + + // Refused. Replayed verbatim, so the transcript and the response agree. + const error = recorded.refusalError ?? "That watch couldn't be started."; + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: refusalMessage(clientRequestId, error), + }); + return { + ok: false, + chatId, + code: (recorded.refusalCode as SubmitWatchErrorCode | null) ?? "internal", + error, + existingId: recorded.refusalExistingId, + }; + }; + + /** + * Record a refusal, then write it under the consent record rather than leaving it to a + * toast the reload forgets. Losing the write means another attempt already settled it. + */ + const refuse = async (refusal: { + code: SubmitWatchErrorCode; + error: string; + existingId?: string | null; + }): Promise => { + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "refused", + refusalCode: refusal.code, + refusalError: refusal.error, + refusalExistingId: refusal.existingId ?? null, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") return replay(winner); + } + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: refusalMessage(clientRequestId, refusal.error), + }); + return { ok: false, chatId, ...refusal }; + }; + + // Step one, before the condition is even read: the ledger row. Its primary key is what + // makes a retry a replay instead of a second operation. + const claim = await claimWatchSubmission(dashboardAgentDb, { + chatId, + clientRequestId, + organizationId, + userId, + projectId: environment.projectId, + environmentId: environment.id, + draftHash: digest, + draft: { spec: draft.spec, followUp: draft.followUp } as unknown as Record, + watchId: generateWatchId(), + }); + + // A chat can by design span environments, so a matching draft is not enough: the row + // has to have been written by this same tenancy, or a staging retry would replay a + // production watch. A mismatch is refused, never replayed. + const recordedScope = claim.submission; + if ( + recordedScope.organizationId !== organizationId || + recordedScope.userId !== userId || + recordedScope.projectId !== environment.projectId || + recordedScope.environmentId !== environment.id + ) { + return { + ok: false, + chatId, + code: "request_conflict", + error: "That request was already submitted somewhere else.", + }; + } + + // A different draft under the same request id is a different request, not a retry. + if (claim.submission.draftHash !== digest) { + return { + ok: false, + chatId, + code: "request_conflict", + error: "That request was already submitted with different settings.", + }; + } + + // Step two, before the watch can exist: a false here means the record is already there + // from an earlier attempt, and a deleted chat is caught by the create below. + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: request, + }); + + let submission = claim.submission; + + // A recorded outcome is replayed. A refusal produced no side effect, so it is the one + // state that may be attempted again — under a fresh reserved id, since the old one may + // already name a cancelled row. + if (submission.state === "refused") { + const reopened = await reopenWatchSubmission(dashboardAgentDb, { + chatId, + clientRequestId, + watchId: generateWatchId(), + }); + if (!reopened) { + const current = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (current && current.state !== "pending") return replay(current); + return refuse({ code: "internal", error: "That watch couldn't be started." }); + } + submission = reopened; + } else if (submission.state !== "pending") { + return replay(submission); + } + + /** Attach the channel, record the outcome, then confirm. A lost race replays the winner. */ + const settleCreated = async (args: { + watchId: string; + unavailable: boolean; + /** The watch was already there: this call adopted it rather than creating it. */ + adopted: boolean; + }): Promise => { + // Attached after the watch exists, and a failure here never fails the creation — it is + // said out loud in the confirmation instead, and recorded so a replay repeats it. + let external: WatchExternalNotification = { status: "not_requested" }; + if (draft.followUp.notifyExternally) { + const subscribed = await subscribe({ userId, environment }); + external = subscribed.ok + ? { status: "enabled" } + : { status: "unavailable", reason: subscribed.reason }; + } + + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "created", + watchId: args.watchId, + unavailable: args.unavailable, + external, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") { + // The winning outcome doesn't name this watch as created, so this watch is an orphan: + // cancel it before replaying, or a refusal would leave a live watch behind. + if (!(winner.state === "created" && winner.watchId === args.watchId)) { + await cancelWatch(dashboardAgentDb, { id: args.watchId, reason: "superseded" }); + } + return replay(winner); + } + } + + return settle({ + confirmation: watchingConfirmation({ + watchId: args.watchId, + unavailable: args.unavailable, + external, + }), + watchId: args.watchId, + repaired: args.adopted, + }); + }; + + // Converge: an attempt that died mid-create left its row under the reserved id. + const reservedWatchId = submission.watchId ?? generateWatchId(); + const reserved = await getWatch(dashboardAgentDb, { id: reservedWatchId }); + if (reserved) { + if (reserved.status === "cancelled") { + // The previous attempt created it and then took it back. The id is spent, so this + // submission can't be completed; a fresh submit gets a fresh request id. + return refuse({ + code: "internal", + error: "The watch couldn't be scheduled. Nothing is being watched.", + }); + } + // `unavailable` isn't recoverable here: it belonged to the attempt that died. + return settleCreated({ watchId: reserved.id, unavailable: false, adopted: true }); + } + + const result = await create({ + environment, + userId, + chatId, + spec: draft.spec, + investigateOnAttention: draft.followUp.investigateOnAttention, + watchId: reservedWatchId, + now: params.now, + deps: params.deps, + }); + + if (!result.ok) { + // Pre-ledger fallback: a submit that started before this ledger existed has no row of + // its own, so an active watch matching the draft is still adopted rather than refused. + // This only ever loads a watch; it never creates one. + if (result.code === "duplicate" && result.existingId) { + const existing = await getWatch(dashboardAgentDb, { id: result.existingId }); + if ( + existing && + existing.chatId === chatId && + existing.status === "active" && + isSameWatchRequest(existing, draft) + ) { + return settleCreated({ watchId: existing.id, unavailable: false, adopted: true }); + } + } + return refuse(result); + } + + if (!result.watching) { + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "immediate", + // No watch exists, so the reserved id is released rather than left dangling. + watchId: null, + immediateResult: result.immediate.result, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") return replay(winner); + } + return settle({ + confirmation: oneShotConfirmation( + result.immediate.result as "satisfied" | "terminal_unsatisfied" + ), + watchId: null, + repaired: false, + }); + } + + return settleCreated({ + watchId: result.watchId, + unavailable: result.unavailable === true, + adopted: false, + }); +} + +/** The two guardrail refusals, worded once for both the pre-check and the insert. */ +function creationGuardrailError( + refusal: + | { error: "limit_reached"; activeCount: number } + | { error: "duplicate"; existingId: string | null } +): CreateDashboardAgentWatchResult { + if (refusal.error === "limit_reached") { + return { + ok: false, + code: "limit_reached", + error: `This chat already has ${MAX_ACTIVE_WATCHES_PER_CHAT} active watches. Cancel one first.`, + }; + } + return { + ok: false, + code: "duplicate", + error: "This chat is already watching that.", + existingId: refusal.existingId, + }; +} + +/** + * Trigger one tick of the watcher task, as the agent's own environment. The token travels + * in the payload, not the database: signing is a pure function of the watch row. + */ +export async function scheduleWatchTick(params: { + watchId: string; + token: string; + delayMinutes: number; + /** The tick generation the scheduled invocation claims. */ + tick: number; +}): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + + await client.tasks.trigger( + WATCH_TASK_ID, + { watchId: params.watchId, token: params.token, apiOrigin, tick: params.tick }, + { + delay: `${params.delayMinutes}m`, + // Keyed on the generation the payload carries, so a retried schedule can't double-tick. + idempotencyKey: `watch:${params.watchId}:tick:${params.tick}`, + // Pin to the same deployed agent version the chat runs on, when set. + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** The task that polls a whole (environment, cadence) group. */ +export const WATCH_BATCH_TASK_ID = "dashboard-agent-watch-batch"; + +/** + * How long a chain may go silent before it is treated as dead and re-armed. Three cadences + * plus two minutes, so a tick's jitter and retries can't trip it. + */ +export function watchBatchStaleMs(cadenceMinutes: number): number { + return cadenceMinutes * 60_000 * 3 + 2 * 60_000; +} + +/** + * Make sure a chain is polling one (environment, cadence) group. A failed trigger un-arms the + * row, since a chain marked running with no run behind it leaves its group unpolled. + */ +export async function armDashboardAgentWatchBatch(params: { + environmentId: string; + cadenceMinutes: number; + now?: Date; + deps?: { + arm?: typeof armWatchBatch; + schedule?: typeof scheduleWatchBatchTick; + stop?: typeof stopWatchBatch; + }; +}): Promise<{ running: boolean }> { + const now = params.now ?? new Date(); + const arm = params.deps?.arm ?? armWatchBatch; + const schedule = params.deps?.schedule ?? scheduleWatchBatchTick; + const stop = params.deps?.stop ?? stopWatchBatch; + + const armed = await arm(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + staleBefore: new Date(now.getTime() - watchBatchStaleMs(params.cadenceMinutes)), + }); + + // A live chain already covers the group. + if (!armed) return { running: true }; + + try { + await schedule({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: armed.epoch, + // A claim lands on `generation + 1`. + tick: armed.generation + 1, + delayMinutes: params.cadenceMinutes, + }); + } catch (error) { + logger.error("Dashboard agent watch: failed to start a batch chain", { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + error, + }); + await stop(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: armed.epoch, + }); + return { running: false }; + } + + return { running: true }; +} + +/** + * Trigger one tick of a batch chain, as the agent's own environment. The chain's token names the + * group and nothing else; the batch check re-authorizes every watch against its own snapshot. + */ +export async function scheduleWatchBatchTick(params: { + environmentId: string; + cadenceMinutes: number; + epoch: number; + tick: number; + delayMinutes: number; +}): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + const token = await mintDashboardAgentWatchBatchToken({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + + await client.tasks.trigger( + WATCH_BATCH_TASK_ID, + { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + apiOrigin, + token, + epoch: params.epoch, + tick: params.tick, + }, + { + delay: `${params.delayMinutes}m`, + // The chain's own key shape, epoch included, so a re-armed chain can't collide + // with its predecessor's keys. + idempotencyKey: `watch-batch:${params.environmentId}:${params.cadenceMinutes}:${params.epoch}:tick:${params.tick}`, + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** + * Hand a resolved watch's wake to the watcher task, since only the agent project may append to a + * chat's `in` stream. Keyed per watch with a short TTL, so a later sweep can still retry. + */ +export async function scheduleWatchDelivery(watch: { id: string; expiresAt: Date }): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + const token = await mintDashboardAgentWatchToken({ + watchId: watch.id, + expiresAt: watch.expiresAt, + }); + + await client.tasks.trigger( + WATCH_TASK_ID, + { watchId: watch.id, token, apiOrigin, tick: 0, deliverOnly: true }, + { + idempotencyKey: `watch:${watch.id}:deliver`, + idempotencyKeyTTL: "10m", + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** + * Delete a chat and end its watches in one transaction, so no live watch is left on an + * invisible chat. Owner-scoped, so a chatId the caller doesn't own deletes nothing. + */ +export async function deleteChatWithWatches(params: { + chatId: string; + userId: string; +}): Promise<{ deleted: boolean; cancelledWatches: number }> { + const result = await softDeleteChat(dashboardAgentDb, params); + return { deleted: result.deleted, cancelledWatches: result.cancelledWatches.length }; +} + +/** Dates are strings because this crosses a loader's JSON boundary. */ +export type ChatWatchChip = { + id: string; + identity: string; + status: WatchStatus; + kind: string; + note: string; + checkEveryMinutes: number; + expiresAt: string; + endedReason: string | null; + /** How the watch ended. Null while active. */ + resolution: WatchResolution | null; + /** What the resolving check observed. */ + observedOutcome: WatchObservedOutcome | null; +}; + +/** + * Active watches for many chats in one query, keyed by chatId. The query layer re-scopes the + * chat ids by org and user, so this is safe with ids from any source. + */ +export async function listActiveWatchesForChats(params: { + chatIds: string[]; + organizationId: string; + userId: string; +}): Promise> { + const byChat = await listActiveWatchesForChatsQuery(dashboardAgentDb, params); + + return Object.fromEntries( + Object.entries(byChat).map(([chatId, watches]) => [ + chatId, + watches.map((watch) => ({ + id: watch.id, + identity: watch.identity, + status: watch.status, + kind: watch.kind, + note: watch.note, + checkEveryMinutes: watch.checkEveryMinutes, + expiresAt: watch.expiresAt.toISOString(), + endedReason: watch.endedReason, + resolution: watch.resolution, + observedOutcome: watch.observedOutcome, + })), + ]) + ); +} + +export function chatBelongsToUser(params: { + chatId: string; + userId: string; + organizationId: string; +}): Promise { + return chatExists(dashboardAgentDb, params); +} + +export type { ChatWatchContext }; + +/** + * Ownership check for a chat, plus its org, the tenancy floor its watches can't leave. No + * project or environment: those come from the authorized request context. + */ +export function resolveChatWatchContext(params: { + chatId: string; + userId: string; +}): Promise { + return getChatWatchContext(dashboardAgentDb, params); +} diff --git a/apps/webapp/app/utils/cspImageOrigins.test.ts b/apps/webapp/app/utils/cspImageOrigins.test.ts index 95391912d90..42f3c33856b 100644 --- a/apps/webapp/app/utils/cspImageOrigins.test.ts +++ b/apps/webapp/app/utils/cspImageOrigins.test.ts @@ -75,16 +75,17 @@ describe("parseCspImageOrigins", () => { }); describe("buildImgSrcDirective", () => { - it("is self, data, blob and the GitHub avatar host by default", () => { + it("is self, data, blob and the SSO avatar hosts by default", () => { expect(buildImgSrcDirective()).toBe( - "img-src 'self' data: blob: https://avatars.githubusercontent.com" + "img-src 'self' data: blob: https://avatars.githubusercontent.com https://lh3.googleusercontent.com" ); }); it("has no wildcard host and no bare scheme host", () => { const directive = buildImgSrcDirective(parseCspImageOrigins("https://sso.example.com").origins); expect(directive).not.toContain("*"); - expect(directive).not.toContain("googleusercontent.com"); + // The avatar hosts are exact origins; a wildcard over them would not be. + expect(directive).not.toContain("*.googleusercontent.com"); expect(directive).not.toMatch(/(^|\s)https?:(\s|$)/); }); diff --git a/apps/webapp/app/utils/cspImageOrigins.ts b/apps/webapp/app/utils/cspImageOrigins.ts index 3afeb306621..de000a22a3a 100644 --- a/apps/webapp/app/utils/cspImageOrigins.ts +++ b/apps/webapp/app/utils/cspImageOrigins.ts @@ -4,12 +4,13 @@ * no wildcard host, no bare scheme, nothing with a path. */ -/** Always allowed: own origin, inline data, object URLs, and the GitHub avatar host. */ +/** Always allowed: own origin, inline data, object URLs, and the SSO avatar hosts. */ export const BASE_IMG_SRC_SOURCES = [ "'self'", "data:", "blob:", "https://avatars.githubusercontent.com", + "https://lh3.googleusercontent.com", ] as const; export type RejectedOrigin = { value: string; reason: string }; diff --git a/apps/webapp/app/v3/alertsWorker.server.ts b/apps/webapp/app/v3/alertsWorker.server.ts index 88637d1c361..a58426821e6 100644 --- a/apps/webapp/app/v3/alertsWorker.server.ts +++ b/apps/webapp/app/v3/alertsWorker.server.ts @@ -5,11 +5,37 @@ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; import { DeliverAlertService } from "./services/alerts/deliverAlert.server"; +import { + DeliverDashboardAgentWatchAlertService, + DeliverDashboardAgentWatchChannelAlertService, +} from "./services/alerts/deliverDashboardAgentWatchAlert.server"; import { DeliverErrorGroupAlertService } from "./services/alerts/deliverErrorGroupAlert.server"; import { ErrorAlertEvaluator } from "./services/alerts/errorAlertEvaluator.server"; +import { + watchObservedOutcomeSchema, + watchResolutionSchema, +} from "@internal/dashboard-agent-contracts"; import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server"; import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server"; +/** The fired watch, as the fan-out and each per-channel delivery carry it. */ +const DashboardAgentWatchAlertPayload = z.object({ + watchId: z.string(), + organizationId: z.string(), + projectId: z.string(), + environmentId: z.string(), + userId: z.string(), + identity: z.string(), + kind: z.string(), + note: z.string(), + firedAt: z.string(), + facts: z.record(z.unknown()), + // Optional so a job enqueued before this deploy still validates and delivers. + resolution: watchResolutionSchema.optional().catch(undefined), + // `.catch` so an unrecognized observation shape degrades instead of dropping the alert. + observed: watchObservedOutcomeSchema.optional().catch(undefined), +}); + function initializeWorker() { const redisOptions = { keyPrefix: "alerts:worker:", @@ -93,6 +119,24 @@ function initializeWorker() { }, logErrors: true, }, + // The fan-out: resolves the channels and enqueues one delivery job each. + "v3.deliverDashboardAgentWatchAlert": { + schema: DashboardAgentWatchAlertPayload, + visibilityTimeoutMs: 60_000, + retry: { + maxAttempts: 3, + }, + logErrors: true, + }, + // One channel's delivery, so a retry only re-sends the channel that failed. + "v3.deliverDashboardAgentWatchAlertChannel": { + schema: DashboardAgentWatchAlertPayload.extend({ channelId: z.string() }), + visibilityTimeoutMs: 60_000, + retry: { + maxAttempts: 3, + }, + logErrors: true, + }, }, concurrency: { workers: env.ALERTS_WORKER_CONCURRENCY_WORKERS, @@ -126,6 +170,14 @@ function initializeWorker() { const service = new DeliverErrorGroupAlertService(); await service.call(payload); }, + "v3.deliverDashboardAgentWatchAlert": async ({ payload }) => { + const service = new DeliverDashboardAgentWatchAlertService(); + await service.call(payload); + }, + "v3.deliverDashboardAgentWatchAlertChannel": async ({ payload }) => { + const service = new DeliverDashboardAgentWatchChannelAlertService(); + await service.call(payload); + }, }, }); diff --git a/apps/webapp/app/v3/commonWorker.server.ts b/apps/webapp/app/v3/commonWorker.server.ts index 65b59209ec5..f104cb4f15d 100644 --- a/apps/webapp/app/v3/commonWorker.server.ts +++ b/apps/webapp/app/v3/commonWorker.server.ts @@ -13,6 +13,10 @@ import { } from "~/services/attio.server"; import { sweepDashboardAgentTurnEvals } from "~/services/dashboardAgentEvalRetention.server"; import { sweepDashboardAgentInvestigations } from "~/services/dashboardAgentInvestigationSweep.server"; +import { + rearmDashboardAgentWatchBatches, + sweepDashboardAgentWatches, +} from "~/services/dashboardAgentWatchSweep.server"; import { logger } from "~/services/logger.server"; import { MembershipDevEnvironmentsSchema, @@ -158,6 +162,16 @@ function initializeWorker() { maxAttempts: 1, }, }, + // The watch backstops: expiry, wake redelivery, retention and dead batch chains. + "dashboardAgent.watchMaintenance": { + schema: CronSchema, + visibilityTimeoutMs: 60_000 * 5, + cron: "*/5 * * * *", + jitterInMs: 30_000, + retry: { + maxAttempts: 1, + }, + }, }, concurrency: { workers: env.COMMON_WORKER_CONCURRENCY_WORKERS, @@ -239,6 +253,30 @@ function initializeWorker() { failure ??= error; } + if (failure) throw failure; + }, + "dashboardAgent.watchMaintenance": async () => { + // Each backstop runs independently; the first failure is rethrown at the end. + let failure: unknown; + + try { + const watches = await sweepDashboardAgentWatches(); + if (watches.overdue > 0 || watches.undelivered > 0 || watches.purged > 0) { + logger.debug("Dashboard agent watch sweep", watches); + } + } catch (error) { + failure ??= error; + } + + try { + const batches = await rearmDashboardAgentWatchBatches(); + if (batches.stale > 0) { + logger.debug("Dashboard agent watch batch re-arm", batches); + } + } catch (error) { + failure ??= error; + } + if (failure) throw failure; }, }, diff --git a/apps/webapp/app/v3/queueDepthSeries.ts b/apps/webapp/app/v3/queueDepthSeries.ts new file mode 100644 index 00000000000..925e3129b34 --- /dev/null +++ b/apps/webapp/app/v3/queueDepthSeries.ts @@ -0,0 +1,49 @@ +/** + * `getQueueDepthSparklines` emits a row only for buckets that reported, so a caller has to place + * every row on the bucket grid itself. Depth is carry-forward filled: no emission means unchanged, + * not zero. Throttled is not filled — only real per-bucket counts tint a bar. + */ + +export type QueueDepthBucketRow = { bucket: string; depth: number; throttled: number }; + +export type QueueDepthGrid = { startMs: number; bucketIntervalMs: number; numBuckets: number }; + +/** Rows placed on the grid by bucket index. Rows outside the window are dropped. */ +export function indexQueueDepthRows( + rows: QueueDepthBucketRow[], + grid: QueueDepthGrid +): Map { + const byIndex = new Map(); + for (const row of rows) { + const bucketMs = Date.parse(row.bucket.replace(" ", "T") + "Z"); + if (Number.isNaN(bucketMs)) continue; + const index = Math.round((bucketMs - grid.startMs) / grid.bucketIntervalMs); + if (index < 0 || index >= grid.numBuckets) continue; + byIndex.set(index, { depth: row.depth, throttled: row.throttled }); + } + return byIndex; +} + +/** A fixed-width series per grid bucket, so a gap can never shift later points in time. */ +export function fillQueueDepthSeries( + byIndex: Map, + numBuckets: number +): { depth: number[]; throttled: number[] } { + const depth: number[] = new Array(numBuckets); + const throttled: number[] = new Array(numBuckets); + let last = 0; + for (let i = 0; i < numBuckets; i++) { + const bucket = byIndex.get(i); + if (bucket !== undefined) last = bucket.depth; + depth[i] = last; + throttled[i] = bucket?.throttled ?? 0; + } + return { depth, throttled }; +} + +export function queueDepthSeries( + rows: QueueDepthBucketRow[], + grid: QueueDepthGrid +): { depth: number[]; throttled: number[] } { + return fillQueueDepthSeries(indexQueueDepthRows(rows, grid), grid.numBuckets); +} diff --git a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts index 20d7c02333a..3ac7f4afb34 100644 --- a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts +++ b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts @@ -391,7 +391,8 @@ export class DeliverAlertService extends BaseService { break; } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } @@ -747,7 +748,8 @@ export class DeliverAlertService extends BaseService { break; } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } @@ -1024,7 +1026,8 @@ export class DeliverAlertService extends BaseService { return; } } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } diff --git a/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts new file mode 100644 index 00000000000..5c4e7a75a3d --- /dev/null +++ b/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts @@ -0,0 +1,506 @@ +import { + type ChatPostMessageArguments, + ErrorCode, + type WebAPIPlatformError, + type WebAPIRateLimitedError, +} from "@slack/web-api"; +import type { WatchObservedOutcome, WatchResolution } from "@internal/dashboard-agent-contracts"; +import { type ProjectAlertChannel } from "@trigger.dev/database"; +import assertNever from "assert-never"; +import { subtle } from "crypto"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { + isIntegrationForService, + type OrganizationIntegrationForService, + OrgIntegrationRepository, +} from "~/models/orgIntegration.server"; +import { + ProjectAlertEmailProperties, + ProjectAlertSlackProperties, + ProjectAlertWebhookProperties, +} from "~/models/projectAlert.server"; +import { mintDashboardAgentAlertUnsubscribeToken } from "~/services/dashboardAgentAlertUnsubscribeToken.server"; +import { + canUseDashboardAgentAlerts, + DASHBOARD_AGENT_WATCH_ALERT_TYPE, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { + presentResolvedWatch, + renderFactLines, + watchNoteLine, +} from "~/presenters/v3/dashboardAgent"; +import { sendAlertEmail } from "~/services/email.server"; +import { logger } from "~/services/logger.server"; +import { decryptSecret } from "~/services/secrets/secretStore.server"; +import { v3RunsPath } from "~/utils/pathBuilder"; +import { alertsWorker } from "~/v3/alertsWorker.server"; +import { safeWebhookFetch } from "./safeWebhookFetch.server"; + +/** + * Deliver a fired dashboard-agent watch to the project's alert channels. No `ProjectAlert` row: + * the job ids are the dedupe, and one job per channel keeps a failing webhook to itself. + */ +export type DashboardAgentWatchAlertPayload = { + watchId: string; + organizationId: string; + projectId: string; + environmentId: string; + userId: string; + identity: string; + kind: string; + note: string; + firedAt: string; + facts: Record; + /** Optional so a job from an older build still delivers, falling back to `condition_met`. */ + resolution?: WatchResolution; + observed?: WatchObservedOutcome; +}; + +/** One channel's delivery: the fan-out payload plus the channel it targets. */ +export type DashboardAgentWatchChannelAlertPayload = DashboardAgentWatchAlertPayload & { + channelId: string; +}; + +/** Bumped when the webhook body's shape changes. */ +const WEBHOOK_VERSION = "2026-08-02"; + +/** Wording comes from the shared presenter, so every surface says the same sentence. */ +function presentAlert(payload: DashboardAgentWatchAlertPayload) { + return presentResolvedWatch({ + kind: payload.kind, + identity: payload.identity, + // Only a met condition fans out today; the fallback keeps an older payload + // from silently presenting as something else. + resolution: payload.resolution ?? "condition_met", + observed: payload.observed ?? null, + }); +} + +class SkipRetryError extends Error {} + +type ResolvedContext = { + environmentName: string; + environmentSlug: string; + organizationSlug: string; + organizationTitle: string; + projectName: string; + projectSlug: string; + projectRef: string; + dashboardLink: string; +}; + +type ResolvedEnvironment = NonNullable>>; + +function findEnvironment(payload: DashboardAgentWatchAlertPayload) { + return $replica.runtimeEnvironment.findFirst({ + where: { id: payload.environmentId, projectId: payload.projectId }, + select: { + type: true, + slug: true, + branchName: true, + project: { + select: { + name: true, + slug: true, + externalRef: true, + organization: { select: { slug: true, title: true } }, + }, + }, + }, + }); +} + +function buildContext(environment: ResolvedEnvironment): ResolvedContext { + return { + environmentName: environment.branchName ?? environment.slug, + environmentSlug: environment.slug, + organizationSlug: environment.project.organization.slug, + organizationTitle: environment.project.organization.title, + projectName: environment.project.name, + projectSlug: environment.project.slug, + projectRef: environment.project.externalRef, + dashboardLink: `${env.APP_ORIGIN}${v3RunsPath( + { slug: environment.project.organization.slug }, + { slug: environment.project.slug }, + { slug: environment.slug } + )}`, + }; +} + +/** The fan-out: gate the watch, then enqueue one delivery job per channel. */ +export class DeliverDashboardAgentWatchAlertService { + async call(payload: DashboardAgentWatchAlertPayload): Promise { + const environment = await findEnvironment(payload); + + if (!environment) { + logger.warn("[DeliverDashboardAgentWatchAlert] Environment not found", { + watchId: payload.watchId, + }); + return; + } + + // Gate at delivery time, not only at subscribe time, so a plan change or a revoked + // feature flag stops the alerts without anyone cleaning up channels. + const gate = await canUseDashboardAgentAlerts({ + userId: payload.userId, + organizationId: payload.organizationId, + organizationSlug: environment.project.organization.slug, + }); + if (!gate.allowed) { + logger.info("[DeliverDashboardAgentWatchAlert] Not allowed for this organization", { + watchId: payload.watchId, + reason: gate.reason, + }); + return; + } + + const channels = await $replica.projectAlertChannel.findMany({ + where: { + projectId: payload.projectId, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + environmentTypes: { has: environment.type }, + }, + select: { id: true }, + }); + + for (const channel of channels) { + await alertsWorker.enqueue({ + // Stable per channel, so a fan-out retry re-enqueues the same job ids + // rather than a second alert per channel. + id: `watch-alert:${payload.watchId}:channel:${channel.id}`, + job: "v3.deliverDashboardAgentWatchAlertChannel", + payload: { ...payload, channelId: channel.id }, + }); + } + } +} + +/** One channel's delivery. A retry here can only re-send this channel. */ +export class DeliverDashboardAgentWatchChannelAlertService { + async call(payload: DashboardAgentWatchChannelAlertPayload): Promise { + // Re-read the channel rather than trusting the fan-out's snapshot: an unsubscribe + // between fan-out and delivery should stop the alert. The primary, since the + // unsubscribe writes there and replica lag would send the mail anyway. + const channel = await prisma.projectAlertChannel.findFirst({ + where: { + id: payload.channelId, + projectId: payload.projectId, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + }, + }); + + if (!channel) { + logger.info("[DeliverDashboardAgentWatchAlert] Channel gone or unsubscribed", { + watchId: payload.watchId, + channelId: payload.channelId, + }); + return; + } + + const environment = await findEnvironment(payload); + + if (!environment) { + logger.warn("[DeliverDashboardAgentWatchAlert] Environment not found", { + watchId: payload.watchId, + }); + return; + } + + const context = buildContext(environment); + + try { + switch (channel.type) { + case "EMAIL": + await this.#sendEmail(channel, payload, context); + break; + case "SLACK": + await this.#sendSlack(channel, payload, context); + break; + case "WEBHOOK": + await this.#sendWebhook(channel, payload, context); + break; + default: + assertNever(channel.type); + } + } catch (error) { + if (error instanceof SkipRetryError) { + logger.warn("[DeliverDashboardAgentWatchAlert] Skipping retry", { + watchId: payload.watchId, + channelId: channel.id, + reason: error.message, + }); + return; + } + throw error; + } + } + + async #sendEmail( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const emailProperties = ProjectAlertEmailProperties.safeParse(channel.properties); + if (!emailProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse email properties", { + issues: emailProperties.error.issues, + }); + return; + } + + const token = await mintDashboardAgentAlertUnsubscribeToken({ + channelId: channel.id, + alertType: DASHBOARD_AGENT_WATCH_ALERT_TYPE, + }); + + const presentation = presentAlert(payload); + + await sendAlertEmail({ + email: "alert-dashboard-agent-watch", + to: emailProperties.data.email, + identity: payload.identity, + kind: payload.kind, + headline: presentation.headline, + tone: presentation.tone, + note: payload.note, + // The sentence that quotes the note, rendered by the shared presenter so the + // email and the Slack message say it the same way. + noteLine: watchNoteLine(payload.note) ?? undefined, + firedAt: payload.firedAt, + facts: factList(payload.facts), + dashboardLink: context.dashboardLink, + unsubscribeLink: `${env.APP_ORIGIN}/resources/dashboard-agent/alerts/${channel.id}/unsubscribe?token=${encodeURIComponent(token)}`, + organization: context.organizationTitle, + project: context.projectName, + environment: context.environmentName, + }); + } + + async #sendSlack( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const slackProperties = ProjectAlertSlackProperties.safeParse(channel.properties); + if (!slackProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse slack properties", { + issues: slackProperties.error.issues, + }); + return; + } + + const integration = slackProperties.data.integrationId + ? await prisma.organizationIntegration.findFirst({ + where: { + id: slackProperties.data.integrationId, + organizationId: payload.organizationId, + }, + include: { tokenReference: true }, + }) + : await prisma.organizationIntegration.findFirst({ + where: { service: "SLACK", organizationId: payload.organizationId }, + orderBy: { createdAt: "desc" }, + include: { tokenReference: true }, + }); + + if (!integration || !isIntegrationForService(integration, "SLACK")) { + logger.error("[DeliverDashboardAgentWatchAlert] Slack integration not found"); + return; + } + + await this.#postSlackMessage(integration, { + channel: slackProperties.data.channelId, + ...this.#buildSlackMessage(payload, context), + } as ChatPostMessageArguments); + } + + async #sendWebhook( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const webhookProperties = ProjectAlertWebhookProperties.safeParse(channel.properties); + if (!webhookProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse webhook properties", { + issues: webhookProperties.error.issues, + }); + return; + } + + const rawPayload = JSON.stringify({ + // Stable across attempts, so a receiver can dedupe a redelivery. Unlike the + // error-group webhook, which mints a nanoid per attempt. + id: `watch:${payload.watchId}:channel:${payload.channelId}`, + created: new Date(payload.firedAt), + webhookVersion: WEBHOOK_VERSION, + type: "alert.dashboard_agent_watch", + object: { + watch: { + id: payload.watchId, + identity: payload.identity, + kind: payload.kind, + note: payload.note, + // `outcome` keeps its two-value encoding for receivers that already parse it; + // `resolution` and `observed` carry the detail. + outcome: "fired", + resolution: payload.resolution ?? "condition_met", + observed: payload.observed ?? null, + firedAt: payload.firedAt, + facts: payload.facts, + }, + environment: { id: payload.environmentId, name: context.environmentName }, + organization: { + id: payload.organizationId, + slug: context.organizationSlug, + name: context.organizationTitle, + }, + project: { + id: payload.projectId, + ref: context.projectRef, + slug: context.projectSlug, + name: context.projectName, + }, + dashboardUrl: context.dashboardLink, + }, + }); + + const secret = await decryptSecret(env.ENCRYPTION_KEY, webhookProperties.data.secret); + const key = await subtle.importKey( + "raw", + Buffer.from(secret, "utf-8"), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"] + ); + const signature = await subtle.sign("HMAC", key, Buffer.from(rawPayload, "utf-8")); + + // Deliver via the SSRF-safe wrapper (see safeWebhookFetch.server.ts). + const response = await safeWebhookFetch(webhookProperties.data.url, { + method: "POST", + headers: { + "content-type": "application/json", + "x-trigger-signature-hmacsha256": Buffer.from(signature).toString("hex"), + }, + body: rawPayload, + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + logger.info("[DeliverDashboardAgentWatchAlert] Failed to send webhook", { + status: response.status, + url: webhookProperties.data.url, + }); + throw new Error(`Failed to send watch alert webhook to ${webhookProperties.data.url}`); + } + } + + async #postSlackMessage( + integration: OrganizationIntegrationForService<"SLACK">, + message: ChatPostMessageArguments + ) { + const client = await OrgIntegrationRepository.getAuthenticatedClientForIntegration( + integration, + { forceBotToken: true } + ); + + try { + return await client.chat.postMessage({ + ...message, + unfurl_links: false, + unfurl_media: false, + }); + } catch (error) { + if (isWebAPIRateLimitedError(error)) { + throw new Error("Slack rate limited"); + } + if (isWebAPIPlatformError(error)) { + const code = (error as WebAPIPlatformError).data.error; + if (code === "invalid_blocks" || code === "account_inactive") { + throw new SkipRetryError(`Slack: ${code}`); + } + throw new Error("Slack platform error"); + } + throw error; + } + } + + #buildSlackMessage( + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): { text: string; blocks: object[] } { + const facts = factList(payload.facts); + const { headline } = presentAlert(payload); + const noteLine = watchNoteLine(payload.note); + + return { + // The notification text is the plain-text rendering, so what a phone shows + // matches what the blocks below say. + text: [`${headline} [${context.environmentName}]`, noteLine, ...renderFactLines(facts)] + .filter(Boolean) + .join("\n"), + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: [`*${headline}* [${context.environmentName}]`, noteLine] + .filter(Boolean) + .join("\n"), + }, + }, + ...(facts.length > 0 + ? [ + { + type: "section", + fields: facts.slice(0, 10).map((fact) => ({ + type: "mrkdwn", + text: `*${fact.label}:*\n${fact.value}`, + })), + }, + ] + : []), + { + type: "actions", + elements: [ + { + type: "button", + text: { type: "plain_text", text: "Open dashboard" }, + url: context.dashboardLink, + style: "primary", + }, + ], + }, + ], + }; + } +} + +/** + * The check's facts, flattened for display. Generic because the bag is per-watch-kind and + * open-ended, and capped so a big bag can't blow up an email or a Slack block. + */ +function factList(facts: Record): Array<{ label: string; value: string }> { + return Object.entries(facts) + .filter(([, value]) => value !== null && value !== undefined && value !== "") + .slice(0, 12) + .map(([key, value]) => ({ + label: humanizeFactKey(key), + value: typeof value === "object" ? JSON.stringify(value).slice(0, 200) : String(value), + })); +} + +function humanizeFactKey(key: string): string { + const spaced = key.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/[_-]+/g, " "); + return spaced.charAt(0).toUpperCase() + spaced.slice(1); +} + +function isWebAPIPlatformError(error: unknown): error is WebAPIPlatformError { + return (error as WebAPIPlatformError).code === ErrorCode.PlatformError; +} + +function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError { + return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError; +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 192f67dc455..67bb8f53c08 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -20,6 +20,7 @@ "db:seed": "tsx seed.ts", "db:seed:ai-spans": "tsx seed-ai-spans.mts", "db:seed:queue-metrics": "tsx seed-queue-metrics.mts", + "scenarios:watch": "tsx seed-watch-scenarios.mts", "upload:sourcemaps": "bash ./upload-sourcemaps.sh", "test": "vitest --no-file-parallelism", "eval:dev": "evalite watch" diff --git a/apps/webapp/seed-watch-scenarios.mts b/apps/webapp/seed-watch-scenarios.mts new file mode 100644 index 00000000000..0e37dda9e85 --- /dev/null +++ b/apps/webapp/seed-watch-scenarios.mts @@ -0,0 +1,868 @@ +/** + * Local watch scenario kit. Run on Node 20; `--help` lists the verbs. + * Walkthroughs: internal-packages/dashboard-agent/GUIDEBOOK.md + */ +import { ClickHouse, TASK_RUN_COLUMNS } from "@internal/clickhouse"; +import type { QueueMetricsRawV1Input } from "@internal/clickhouse"; +import { randomUUID } from "node:crypto"; +// oxlint-disable import/default -- deliberate CommonJS interop: this entry is ESM, +// tsx compiles the app's `.ts` to CommonJS, and an ESM importer reaches a CommonJS +// module's exports only through `default`. +import dbServer from "./app/db.server"; +import errorFingerprinting from "./app/utils/errorFingerprinting"; +import eventCommon from "./app/v3/eventRepository/common.server"; + +const { prisma } = dbServer; +const { calculateErrorFingerprint } = errorFingerprinting; +const { generateTraceId, generateSpanId } = eventCommon; + +const APP_ORIGIN = process.env.APP_ORIGIN ?? "http://localhost:3030"; + +const DEFAULT_RUN_SECONDS = 60; +const DEFAULT_FAIL_TASK = "slow-fail"; +const DEFAULT_SUCCEED_TASK = "slow-succeed"; + +/** One fixed shape: the fingerprint has to stay stable across runs. */ +const SCENARIO_ERROR = { + type: "BUILT_IN_ERROR", + name: "ProviderError", + message: "429 Too Many Requests (rate_limit_exceeded)", + stackTrace: `ProviderError: 429 Too Many Requests (rate_limit_exceeded) + at sendEmail (src/trigger/scenarioKit.ts:31:11) + at run (src/trigger/scenarioKit.ts:18:5)`, +}; +const SCENARIO_ERROR_FINGERPRINT = calculateErrorFingerprint(SCENARIO_ERROR); +const SCENARIO_TASK_ID = "scenario-kit-task"; +const SCENARIO_QUEUE_NAME = "scenario-kit"; + +function fail(message: string): never { + console.error(`\n${message}\n`); + process.exit(1); +} + +function parseFlags(argv: string[]): { positional: string[]; flags: Record } { + const positional: string[] = []; + const flags: Record = {}; + for (let i = 0; i < argv.length; i++) { + const token = argv[i]; + // `pnpm run